Merge upstream commit '2e646457' into master
* commit '2e646457':
Document example dyndep use cases
Document `dyndep` binding behavior and the dyndep file format
query: load dyndep files for queried edges
graph: load dyndep files
clean: remove outputs specified by dyndep files
clean: remove unnecessary Cleaner constructor variant
Teach builder to load dyndep files when they are ready
Teach RecomputeDirty to load dyndep files that are ready
Teach DependencyScan to load a dyndep file
Add a "dyndep" reserved binding to the manifest format
Add a parser for a new "dyndep" file format
Explicitly avoid repeat deps loading
Make a Builder optionally available to Plan
Factor out output edge ready check from Plan::NodeFinished
Factor out edge marking logic from Plan::AddSubTarget
Teach FakeCommandRunner to support multiple active commands
Allow EdgeFinished and NodeFinished to fail with errors
Assert precondition in BuildStatus::BuildEdgeStarted
Factor out a base class of ManifestParser
ManifestParser: Fix typo {expectd => expected}
This brings in the upstream dyndep feature into our ninja fork. The
majority of the conflicts were from our parallel manifest parser, as
that changed the Lexer and all of the BindingEnv classes, which required
some rewrites. Upstream shares a base parsing class between the
ManifestParser and DyndepParser, that doesn't make sense with our new
parser, so DyndepParser is the only user of the Parser base class.
Colin's validation nodes change also interacts with dyndeps, I pulled in
the new code from his github pull request.
Test: builtin ninja tests
Test: treehugger with updated prebuilt
Change-Id: I90b059d0c28a353d50e74efc9097cc39d88a5583
diff --git a/Android.bp b/Android.bp
new file mode 100644
index 0000000..7f14db6
--- /dev/null
+++ b/Android.bp
@@ -0,0 +1,137 @@
+// Copyright 2016 Google Inc. All rights reserved
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+cc_defaults {
+ name: "ninja_defaults",
+ cflags: [
+ "-Wall",
+ "-Wextra",
+ "-Wno-deprecated",
+ "-Wno-missing-field-initializers",
+ "-Wno-unused-parameter",
+ "-fno-exceptions",
+ "-fvisibility=hidden",
+ "-DNINJA_PYTHON=\"python\"",
+ "-DNINJA_HAVE_BROWSE",
+ "-UNDEBUG",
+ ],
+ target: {
+ linux_glibc: {
+ cflags: ["-DUSE_PPOLL"],
+ }
+ }
+}
+
+genrule {
+ name: "ninja_browse_py",
+
+ cmd: "$(location src/inline.sh) kBrowsePy <$(in) >$(out)",
+ tool_files: ["src/inline.sh"],
+
+ srcs: ["src/browse.py"],
+ out: ["build/browse_py.h"],
+}
+
+cc_library_host_static {
+ name: "libninja",
+ defaults: ["ninja_defaults"],
+ generated_headers: ["ninja_browse_py"],
+ srcs: [
+ "src/build.cc",
+ "src/build_log.cc",
+ "src/clean.cc",
+ "src/clparser.cc",
+ "src/debug_flags.cc",
+ "src/depfile_parser.cc",
+ "src/deps_log.cc",
+ "src/disk_interface.cc",
+ "src/dyndep.cc",
+ "src/dyndep_parser.cc",
+ "src/edit_distance.cc",
+ "src/eval_env.cc",
+ "src/graph.cc",
+ "src/graphviz.cc",
+ "src/lexer.cc",
+ "src/line_printer.cc",
+ "src/manifest_chunk_parser.cc",
+ "src/manifest_parser.cc",
+ "src/metrics.cc",
+ "src/parser.cc",
+ "src/proto.cc",
+ "src/state.cc",
+ "src/status.cc",
+ "src/thread_pool.cc",
+ "src/util.cc",
+ "src/version.cc",
+ "src/browse.cc",
+ "src/subprocess-posix.cc",
+ ],
+}
+
+cc_binary_host {
+ name: "ninja",
+ defaults: ["ninja_defaults"],
+ srcs: ["src/ninja.cc"],
+ static_libs: ["libninja"],
+
+ // Use jemalloc for better multithreaded allocation performance. e.g. Using
+ // jemalloc can improve the overall runtime by 10x vs the default allocator.
+ target: {
+ linux_glibc: {
+ shared_libs: ["libjemalloc"],
+ },
+ },
+}
+
+cc_test_host {
+ name: "ninja_test",
+ defaults: ["ninja_defaults"],
+ static_libs: ["libninja"],
+ gtest: false,
+ srcs: [
+ "src/build_log_test.cc",
+ "src/build_test.cc",
+ "src/clean_test.cc",
+ "src/clparser_test.cc",
+ "src/depfile_parser_test.cc",
+ "src/deps_log_test.cc",
+ "src/disk_interface_test.cc",
+ "src/dyndep_parser_test.cc",
+ "src/edit_distance_test.cc",
+ "src/graph_test.cc",
+ "src/lexer_test.cc",
+ "src/manifest_parser_test.cc",
+ "src/ninja_test.cc",
+ "src/state_test.cc",
+ "src/status_test.cc",
+ "src/subprocess_test.cc",
+ "src/test.cc",
+ "src/util_test.cc",
+ ],
+}
+
+cc_test_host {
+ name: "ninja_tests",
+ gtest: false,
+ defaults: ["ninja_defaults"],
+ test_per_src: true,
+ static_libs: ["libninja"],
+ srcs: [
+ "src/build_log_perftest.cc",
+ "src/canon_perftest.cc",
+ "src/depfile_parser_perftest.cc",
+ "src/hash_collision_bench.cc",
+ "src/manifest_parser_perftest.cc",
+ ],
+}
diff --git a/configure.py b/configure.py
index 850bb98..6c0acda 100755
--- a/configure.py
+++ b/configure.py
@@ -272,6 +272,8 @@
def src(filename):
return os.path.join('$root', 'src', filename)
+def frontend(filename):
+ return os.path.join('$root', 'frontend', filename)
def built(filename):
return os.path.join('$builddir', filename)
def doc(filename):
@@ -328,7 +330,8 @@
cflags += ['/Ox', '/DNDEBUG', '/GL']
ldflags += ['/LTCG', '/OPT:REF', '/OPT:ICF']
else:
- cflags = ['-g', '-Wall', '-Wextra',
+ cflags = ['-std=c++11',
+ '-g', '-Wall', '-Wextra',
'-Wno-deprecated',
'-Wno-missing-field-initializers',
'-Wno-unused-parameter',
@@ -364,7 +367,10 @@
libs = []
-if platform.is_mingw():
+if platform.is_linux():
+ cflags.append('-pthread')
+ ldflags.append('-pthread')
+elif platform.is_mingw():
cflags.remove('-fvisibility=hidden');
ldflags.append('-static')
elif platform.is_solaris():
@@ -484,6 +490,43 @@
"changes to src/*.in.cc will not affect your build.")
n.newline()
+n.comment('the proto descriptor is generated using protoc.')
+def has_protoc():
+ try:
+ proc = subprocess.Popen(['protoc', '--version'], stdout=subprocess.PIPE)
+ return proc.communicate()[0].startswith("libprotoc")
+ except OSError:
+ return False
+
+def can_generate_proto_header():
+ try:
+ tool = os.path.join(sourcedir, 'misc', 'generate_proto_header.py')
+ proc = subprocess.Popen([tool, '--probe'], stdout=subprocess.PIPE)
+ return proc.communicate()[0].startswith("ok")
+ except OSError:
+ return False
+
+if has_protoc() and can_generate_proto_header():
+ # Use protoc to write out frontend.proto converted to a descriptor proto
+ n.rule('protoc',
+ command='protoc $in -o $out',
+ description='PROTOC $out')
+ n.build(frontend('frontend.pb'), 'protoc', src('frontend.proto'))
+
+ # Use generate_proto_header.py to read in the descriptor proto and write
+ # a header containing field numbers and types.
+ n.rule('generate_proto_header',
+ command='$tool $in $out',
+ description='GEN $out')
+ # Generate the .h file in the source directory so we can check them in.
+ tool = os.path.join(sourcedir, 'misc', 'generate_proto_header.py')
+ n.build(src('frontend.pb.h'), 'generate_proto_header', frontend('frontend.pb'),
+ implicit=[tool], variables=[('tool', tool)])
+else:
+ print("warning: A version of protoc or the python protobuf library was not found; "
+ "changes to src/frontend.proto will not affect your build.")
+n.newline()
+
n.comment('Core source files all build into ninja library.')
cxxvariables = []
if platform.is_msvc():
@@ -504,14 +547,18 @@
'graphviz',
'lexer',
'line_printer',
+ 'manifest_chunk_parser',
'manifest_parser',
'metrics',
'parser',
+ 'proto',
'state',
+ 'status',
'string_piece_util',
+ 'thread_pool',
'util',
'version']:
- objs += cxx(name, variables=cxxvariables)
+ objs += cxx(name, order_only=src('frontend.pb.h'), variables=cxxvariables)
if platform.is_windows():
for name in ['subprocess-win32',
'includes_normalize-win32',
@@ -574,6 +621,7 @@
'manifest_parser_test',
'ninja_test',
'state_test',
+ 'status_test',
'string_piece_util_test',
'subprocess_test',
'test',
diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc
index c9309ad..54a36fd 100644
--- a/doc/manual.asciidoc
+++ b/doc/manual.asciidoc
@@ -1,5 +1,6 @@
The Ninja build system
======================
+v1.7.1, Apr 2016
Introduction
@@ -695,6 +696,8 @@
Order-only dependencies may be tacked on the end with +||
_dependency1_ _dependency2_+. (See <<ref_dependencies,the reference on
dependency types>>.)
+ Validations may be taked on the end with +|@ _validation1_ _validation2_+.
+ (See <<validations,the reference on validations>>.)
+
Implicit outputs _(available since Ninja 1.7)_ may be added before
the `:` with +| _output1_ _output2_+ and do not appear in `$out`.
@@ -821,6 +824,31 @@
discovered dependency information will be loaded from the file.
See the <<ref_dyndep,dynamic dependencies>> section for details.
+`phony_output`:: _(Android-specific patch)_ if present, Ninja will not attempt
+ to look for them on disk. These rules are considered always dirty, and will
+ run every time they're depended upon. This behavior is very similar to Make's
+ `.PHONY` concept.
++
+This can be similar to the `phony` rule, but can have an attached `command`.
+`phony` rules are also only considered dirty in two cases: if their inputs are
+dirty, or if they have no inputs and a file with the same name does not exist on
+disk.
++
+Other build rules may not depend on `phony_output` rules unless they are also
+`phony_output`, so that it's not possible to accidentally cause everything to
+rebuild on every run.
++
+When `-o usesphonyoutputs=yes` is set on the ninja command line, it becomes an
+error for a `phony` rule to cause rebuilds, so that users can be found and
+migrated.
++
+Properly using `phony_output` and turning on `-o usesphonyoutputs=yes` allows
+the `-w outputdir={err,warn}` (consider output files that are directories as
+errors/warnings), `-w missingoutfile={err,warn}` (error/warn when an output file
+does not exist after a successful rule execution), and `-w oldoutput={err,warn}`
+(error/warn when an output file is not updated after a successful non-restat
+rule execution) flags to function.
+
`generator`:: if present, specifies that this rule is used to
re-invoke the generator program. Files built using `generator`
rules are treated specially in two ways: firstly, they will not be
@@ -952,6 +980,31 @@
File paths are compared as is, which means that an absolute path and a
relative path, pointing to the same file, are considered different by Ninja.
+[[validations]]
+Validations
+~~~~~~~~~~~
+Validations listed on the build line cause the specified files to be
+added to the top level of the build graph (as if they were specified
+on the Ninja command line) whenever the build line is a transitive
+dependency of one of the targets specified on the command line or a
+default target.
+
+Validations are added to the build graph regardless of whether the output
+files of the build statement are dirty are not, and the dirty state of
+the build statement that outputs the file being used as a validation
+has no effect on the dirty state of the build statement that requested it.
+
+A build edge can list another build edge as a validation even if the second
+edge depends on the first.
+
+Validations are designed to handle rules that perform error checking but
+don't produce any artifacts needed by the build, for example static
+analysis tools. Marking the static analysis rule as an implicit input
+of the main build rule of the source files or of the rules that depend
+on the main build rule would slow down the critical path of the build,
+but using a validation would allow the build to proceed in parallel with
+the static analysis rule once the main build rule is complete.
+
Variable expansion
~~~~~~~~~~~~~~~~~~
diff --git a/frontend/FRONTEND.md b/frontend/FRONTEND.md
new file mode 100644
index 0000000..5ff6e32
--- /dev/null
+++ b/frontend/FRONTEND.md
@@ -0,0 +1,51 @@
+Ninja Frontend Interface
+========================
+
+Ninja can use another program as a frontend to display build status information.
+This document describes the interface between Ninja and the frontend.
+
+Connecting
+----------
+
+The frontend is passed to Ninja using a --frontend argument. The argument is
+executed the same as a build rule Ninja, wrapped in `sh -c` on Linux. The
+frontend will be executed with the read end of a pipe open on file descriptor
+`3`.
+
+Ninja will pass [Protocol Buffers](https://developers.google.com/protocol-buffers/) generated from src/frontend.proto.
+
+stdin/stdout/stderr
+-------------------
+
+The frontend will have stdin, stdout, and stderr connected to the same file
+descriptors as Ninja. The frontend MAY read from stdin, however, if it does,
+it MUST NOT read from stdin whenever a job in the console pool is running,
+from when an `EdgeStarted` message is received with the `use_console` value
+set to `true`, to when an `EdgeFinished` message is received with the same value
+for `id`. Console rules may write directly to the same stdout/stderr as the
+frontend.
+
+Exiting
+-------
+
+The frontend MUST exit when the input pipe on fd `3` is closed. When a build
+finishes, either successfully, due to error, or on interrupt, Ninja will close
+the pipe and then block until the frontend exits.
+
+Experimenting with frontends
+----------------------------
+
+To run Ninja with a frontend that mimics the behavior of Ninja's normal output:
+```
+$ ./ninja --frontend=frontend/native.py
+```
+
+To save serialized output to a file:
+```
+$ ./ninja --frontend='cat <&3 > ninja.pb' all
+```
+
+To run a frontend with serialized input from a file:
+```
+$ frontend/native.py 3< ninja.pb
+```
diff --git a/frontend/dump.py b/frontend/dump.py
new file mode 100755
index 0000000..d276fd0
--- /dev/null
+++ b/frontend/dump.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import sys
+
+import frontend
+
+def main():
+ if len(sys.argv) >= 2:
+ f = open(sys.argv[1], 'rb')
+ else:
+ f = frontend.default_reader()
+
+ for msg in frontend.Frontend(f):
+ print('---------------------------------')
+ sys.stdout.write(str(msg))
+
+if __name__ == '__main__':
+ main()
+
diff --git a/frontend/frontend.pb b/frontend/frontend.pb
new file mode 100644
index 0000000..718f173
--- /dev/null
+++ b/frontend/frontend.pb
Binary files differ
diff --git a/frontend/frontend.py b/frontend/frontend.py
new file mode 100755
index 0000000..ac8fa25
--- /dev/null
+++ b/frontend/frontend.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+"""Ninja frontend interface.
+
+This module implements a Ninja frontend interface that delegates handling each
+message to a handler object
+"""
+
+import os
+import sys
+
+import google.protobuf.descriptor_pb2
+import google.protobuf.message_factory
+
+def default_reader():
+ fd = 3
+ return os.fdopen(fd, 'rb', 0)
+
+class Frontend(object):
+ """Generator class that parses length-delimited ninja status messages
+ through a ninja frontend interface.
+ """
+
+ def __init__(self, reader=default_reader()):
+ self.reader = reader
+ self.status_class = self.get_status_proto()
+
+ def get_status_proto(self):
+ fd_set = google.protobuf.descriptor_pb2.FileDescriptorSet()
+ descriptor = os.path.join(os.path.dirname(__file__), 'frontend.pb')
+ with open(descriptor, 'rb') as f:
+ fd_set.ParseFromString(f.read())
+
+ if len(fd_set.file) != 1:
+ raise RuntimeError('expected exactly one file descriptor in ' + descriptor)
+
+ messages = google.protobuf.message_factory.GetMessages(fd_set.file)
+ return messages['ninja.Status']
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return self.next()
+
+ def next(self):
+ size = 0
+ shift = 0
+ while True:
+ byte = bytearray(self.reader.read(1))
+ if len(byte) == 0:
+ raise StopIteration()
+
+ byte = byte[0]
+ size += (byte & 0x7f) << (shift * 7)
+ if (byte & 0x80) == 0:
+ break
+ shift += 1
+ if shift > 4:
+ raise RuntimeError('Expected varint32 length-delimeted message')
+
+ message = self.reader.read(size)
+ if len(message) != size:
+ raise EOFError('Unexpected EOF reading %d bytes' % size)
+
+ return self.status_class.FromString(message)
diff --git a/frontend/native.py b/frontend/native.py
new file mode 100755
index 0000000..4862268
--- /dev/null
+++ b/frontend/native.py
@@ -0,0 +1,303 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import collections
+import ctypes
+import os
+import re
+import struct
+import sys
+
+import frontend
+
+class SlidingRateInfo(object):
+ def __init__(self, n=32):
+ self.rate = -1
+ self.last_update = -1
+ self.times = collections.deque(maxlen=n)
+
+ def update_rate(self, update_hint, time_millis):
+ if update_hint == self.last_update:
+ return
+
+ self.last_update = update_hint
+
+ if len(self.times) == self.times.maxlen:
+ self.times.popleft()
+ self.times.append(time_millis)
+ if self.times[-1] != self.times[0]:
+ self.rate = len(self.times) / ((self.times[-1] - self.times[0]) / 1e3)
+
+strip_ansi_re = re.compile(r'\x1B\[[^a-zA-Z]*[a-zA-Z]')
+def strip_ansi_escape_codes(output):
+ return strip_ansi_re.sub('', output)
+
+class NinjaNativeFrontend:
+ def __init__(self):
+ self.total_edges = 0
+ self.running_edges = 0
+ self.started_edges = 0
+ self.finished_edges = 0
+ self.running = {}
+
+ self.time_millis = 0
+
+ self.progress_status_format = os.getenv('NINJA_STATUS', '[%f/%t] ')
+ self.current_rate = SlidingRateInfo()
+ self.console_locked = False
+
+ self.printer = LinePrinter()
+ self.verbose = False
+
+ def handle(self, msg):
+ handled = False
+ if msg.HasField('total_edges'):
+ handled = True
+ self.total_edges = msg.total_edges.total_edges
+
+ if msg.HasField('build_started'):
+ handled = True
+ self.verbose = msg.build_started.verbose
+ self.current_rate = SlidingRateInfo(msg.build_started.parallelism)
+ self.running_edges = 0
+ self.started_edges = 0
+ self.finished_edges = 0
+ self.running = {}
+
+ if msg.HasField('build_finished'):
+ handled = True
+ self.printer.set_console_locked(False)
+ self.printer.print_on_new_line('')
+
+ if msg.HasField('edge_started'):
+ handled = True
+ self.started_edges += 1
+ self.running_edges += 1
+ self.running[msg.edge_started.id] = msg.edge_started
+ self.time_millis = msg.edge_started.start_time
+ if msg.edge_started.console or self.printer.smart_terminal:
+ self.print_status(msg.edge_started)
+ if msg.edge_started.console:
+ self.printer.set_console_locked(True)
+
+ if msg.HasField('edge_finished'):
+ handled = True
+ self.finished_edges += 1
+ self.time_millis = msg.edge_finished.end_time
+
+ edge_started = self.running[msg.edge_finished.id]
+
+ if edge_started.console:
+ self.printer.set_console_locked(False)
+
+ if not edge_started.console:
+ self.print_status(edge_started)
+
+ self.running_edges -= 1
+ del self.running[msg.edge_finished.id]
+
+ # Print the command that is spewing before printing its output.
+ if msg.edge_finished.status != 0:
+ self.printer.print_on_new_line('FAILED: ' + ' '.join(edge_started.outputs))
+ self.printer.print_on_new_line(edge_started.command)
+
+ # ninja sets stdout and stderr of subprocesses to a pipe, to be able to
+ # check if the output is empty. Some compilers, e.g. clang, check
+ # isatty(stderr) to decide if they should print colored output.
+ # To make it possible to use colored output with ninja, subprocesses should
+ # be run with a flag that forces them to always print color escape codes.
+ # To make sure these escape codes don't show up in a file if ninja's output
+ # is piped to a file, ninja strips ansi escape codes again if it's not
+ # writing to a |smart_terminal_|.
+ # (Launching subprocesses in pseudo ttys doesn't work because there are
+ # only a few hundred available on some systems, and ninja can launch
+ # thousands of parallel compile commands.)
+ # TODO: There should be a flag to disable escape code stripping.
+ if msg.edge_finished.output != '':
+ if not self.printer.smart_terminal:
+ msg.edge_finished.output = strip_ansi_escape_codes(msg.edge_finished.output)
+ self.printer.print_on_new_line(msg.edge_finished.output)
+
+ if msg.HasField('message'):
+ handled = True
+ # TODO(colincross): get the enum values from proto
+ if msg.message.level == 0:
+ prefix = 'ninja: '
+ elif msg.message.level == 1:
+ prefix = 'ninja: warning: '
+ elif msg.message.level == 2:
+ prefix = 'ninja: error: '
+ else
+ prefix = ''
+ self.printer.print_line(prefix + msg.message.message, LinePrinter.LINE_FULL)
+
+ if not handled:
+ pass
+
+
+ def format_progress_status(self, fmt):
+ out = ''
+ fmt_iter = iter(fmt)
+ for c in fmt_iter:
+ if c == '%':
+ c = next(fmt_iter)
+ if c == '%':
+ out += c
+ elif c == 's':
+ out += str(self.started_edges)
+ elif c == 't':
+ out += str(self.total_edges)
+ elif c == 'r':
+ out += str(self.running_edges)
+ elif c == 'u':
+ out += str(self.total_edges - self.started_edges)
+ elif c == 'f':
+ out += str(self.finished_edges)
+ elif c == 'o':
+ if self.time_millis > 0:
+ rate = self.finished_edges / (self.time_millis / 1e3)
+ out += '{:.1f}'.format(rate)
+ else:
+ out += '?'
+ elif c == 'c':
+ self.current_rate.update_rate(self.finished_edges, self.time_millis)
+ if self.current_rate.rate == -1:
+ out += '?'
+ else:
+ out += '{:.1f}'.format(self.current_rate.rate)
+ elif c == 'p':
+ out += '{:3d}%'.format((100 * self.finished_edges) // self.total_edges)
+ elif c == 'e':
+ out += '{:.3f}'.format(self.time_millis / 1e3)
+ else:
+ raise RuntimeError('unknown placeholder '' + c +'' in $NINJA_STATUS')
+ else:
+ out += c
+ return out
+
+ def print_status(self, edge_started):
+ to_print = edge_started.desc
+ if self.verbose or to_print == '':
+ to_print = edge_started.command
+
+ to_print = self.format_progress_status(self.progress_status_format) + to_print
+
+ self.printer.print_line(to_print, LinePrinter.LINE_FULL if self.verbose else LinePrinter.LINE_ELIDE)
+
+
+def elide_middle(status, width):
+ margin = 3 # Space for '...'.
+ if len(status) + margin > width:
+ elide_size = (width - margin) // 2
+ status = status[0:elide_size] + '...' + status[-elide_size:]
+ return status
+
+class LinePrinter(object):
+ LINE_FULL = 1
+ LINE_ELIDE = 2
+
+ def __init__(self):
+ # Whether we can do fancy terminal control codes.
+ self.smart_terminal = False
+
+ # Whether the caret is at the beginning of a blank line.
+ self.have_blank_line = True
+
+ # Whether console is locked.
+ self.console_locked = False
+
+ # Buffered current line while console is locked.
+ self.line_buffer = ''
+
+ # Buffered line type while console is locked.
+ self.line_type = self.LINE_FULL
+
+ # Buffered console output while console is locked.
+ self.output_buffer = ''
+
+ if os.name == 'windows':
+ STD_OUTPUT_HANDLE = -11
+ self.console = ctypes.windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE)
+ csbi = ctypes.create_string_buffer(22)
+ self.smart_terminal = ctypes.windll.kernel32.GetConsoleScreenBufferInfo(self.console, csbi)
+ else:
+ term = os.getenv('TERM')
+ self.smart_terminal = os.isatty(1) and term != '' and term != 'dumb'
+
+ def print_line(self, to_print, line_type):
+ if self.console_locked:
+ self.line_buffer = to_print
+ self.line_type = line_type
+
+ if self.smart_terminal:
+ sys.stdout.write('\r') # Print over previous line, if any.
+
+ if self.smart_terminal and line_type == self.LINE_ELIDE:
+ if os.name == 'windows':
+ csbi = ctypes.create_string_buffer(22)
+ ctypes.windll.kernel32.GetConsoleScreenBufferInfo(self.console, csbi)
+ (cols, rows) = struct.unpack('hh', csbi.raw)
+ to_print = elide_middle(to_print, cols)
+ # TODO: windows support
+ # We don't want to have the cursor spamming back and forth, so instead of
+ # printf use WriteConsoleOutput which updates the contents of the buffer,
+ # but doesn't move the cursor position.
+ sys.stdout.write(to_print)
+ sys.stdout.flush()
+ else:
+ # Limit output to width of the terminal if provided so we don't cause
+ # line-wrapping.
+ import fcntl, termios
+ winsize = fcntl.ioctl(0, termios.TIOCGWINSZ, '\0'*4)
+ (rows, cols) = struct.unpack('hh', winsize)
+ to_print = elide_middle(to_print, cols)
+ sys.stdout.write(to_print)
+ sys.stdout.write('\x1B[K') # Clear to end of line.
+ sys.stdout.flush()
+
+ self.have_blank_line = False
+ else:
+ sys.stdout.write(to_print + '\n')
+ sys.stdout.flush()
+
+ def print_or_buffer(self, to_print):
+ if self.console_locked:
+ self.output_buffer += to_print
+ else:
+ sys.stdout.write(to_print)
+ sys.stdout.flush()
+
+ def print_on_new_line(self, to_print):
+ if self.console_locked or self.line_buffer != '':
+ self.output_buffer += self.line_buffer + '\n'
+ self.line_buffer = ''
+ if not self.have_blank_line:
+ self.print_or_buffer('\n')
+ if to_print != '':
+ self.print_or_buffer(to_print)
+ self.have_blank_line = to_print == '' or to_print[0] == '\n'
+
+ def set_console_locked(self, locked):
+ if locked == self.console_locked:
+ return
+
+ if locked:
+ self.print_on_new_line('\n')
+
+ self.console_locked = locked
+
+ if not locked:
+ self.print_on_new_line(self.output_buffer)
+ if self.line_buffer != '':
+ self.print_line(self.line_buffer, self.line_type)
+ self.output_buffer = ''
+ self.line_buffer = ''
+
+def main():
+ native = NinjaNativeFrontend()
+ for msg in frontend.Frontend():
+ native.handle(msg)
+
+if __name__ == '__main__':
+ main()
diff --git a/misc/generate_proto_header.py b/misc/generate_proto_header.py
new file mode 100755
index 0000000..a07eda6
--- /dev/null
+++ b/misc/generate_proto_header.py
@@ -0,0 +1,491 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Write-only protobuf C++ code generator for a minimal runtime
+
+This script uses a descriptor proto generated by protoc and the descriptor_pb2
+distributed with python protobuf to iterate through the fields in a proto
+and write out simple C++ data objects with serialization methods. The generated
+files depend on a tiny runtime implemented in src/proto.h and src/proto.cc.
+"""
+
+from __future__ import print_function
+
+import os.path
+import re
+import sys
+
+import google.protobuf.descriptor_pb2
+import google.protobuf.descriptor
+
+FieldDescriptor = google.protobuf.descriptor.FieldDescriptor
+
+CPP_TYPE_MAP = {
+ FieldDescriptor.CPPTYPE_INT32: 'int32_t',
+ FieldDescriptor.CPPTYPE_INT64: 'int64_t',
+ FieldDescriptor.CPPTYPE_UINT32: 'uint32_t',
+ FieldDescriptor.CPPTYPE_UINT64: 'uint64_t',
+ FieldDescriptor.CPPTYPE_DOUBLE: 'double',
+ FieldDescriptor.CPPTYPE_FLOAT: 'float',
+ FieldDescriptor.CPPTYPE_BOOL: 'bool',
+ FieldDescriptor.CPPTYPE_STRING: 'std::string',
+}
+
+ENCODING_MAP = {
+ FieldDescriptor.TYPE_INT32:
+ ('VarintSize32SignExtended', 'WriteVarint32SignExtended', None),
+ FieldDescriptor.TYPE_INT64:
+ ('VarintSize64', 'WriteVarint64', None),
+ FieldDescriptor.TYPE_UINT32:
+ ('VarintSize32', 'WriteVarint32', None),
+ FieldDescriptor.TYPE_UINT64:
+ ('VarintSize64', 'WriteVarint64', None),
+ FieldDescriptor.TYPE_SINT32:
+ ('VarintSize32', 'WriteVarint32', 'ZigZagEncode32'),
+ FieldDescriptor.TYPE_SINT64:
+ ('VarintSize64', 'WriteVarint64', 'ZigZagEncode64'),
+ FieldDescriptor.TYPE_BOOL:
+ ('VarintSizeBool', 'WriteVarint32', None),
+ FieldDescriptor.TYPE_ENUM:
+ ('VarintSize32SignExtended', 'WriteVarint32SignExtended',
+ 'static_cast<int32_t>'),
+ FieldDescriptor.TYPE_FIXED64:
+ ('FixedSize64', 'WriteFixed64', None),
+ FieldDescriptor.TYPE_SFIXED64:
+ ('FixedSize64', 'WriteFixed64', 'static_cast<uint64_t>'),
+ FieldDescriptor.TYPE_DOUBLE:
+ ('FixedSize64', 'WriteFixed64', 'static_cast<uint64_t>'),
+ FieldDescriptor.TYPE_STRING:
+ ('StringSize', 'WriteString', None),
+ FieldDescriptor.TYPE_BYTES:
+ ('StringSize', 'WriteString', None),
+ FieldDescriptor.TYPE_FIXED32:
+ ('FixedSize32', 'WriteFixed32', None),
+ FieldDescriptor.TYPE_SFIXED32:
+ ('FixedSize32', 'WriteFixed32', 'static_cast<uint32_t>'),
+ FieldDescriptor.TYPE_FLOAT:
+ ('FixedSize32', 'WriteFixed32', 'static_cast<uint32_t>'),
+}
+
+ZIGZAG_LIST = (
+ FieldDescriptor.TYPE_SINT32,
+ FieldDescriptor.TYPE_SINT64,
+ FieldDescriptor.TYPE_SFIXED64,
+ FieldDescriptor.TYPE_SFIXED32,
+)
+
+def field_type_to_cpp_type(field):
+ """Convert a proto field object to its C++ type"""
+ if field.type_name != '':
+ cpp_type = field.type_name.replace('.', '::')
+ else:
+ cpp_type = FieldDescriptor.ProtoTypeToCppProtoType(field.type)
+ cpp_type = CPP_TYPE_MAP[cpp_type]
+ return cpp_type
+
+class Generator:
+ def __init__(self, out):
+ self.w = Writer(out)
+
+ """Class to generate C++ code for a proto descriptor"""
+ def write_enum(self, enum):
+ """Write a proto enum object to the generated file"""
+ self.w.writelines("""
+ enum %(name)s {
+ """ % {
+ 'name': enum.name,
+ })
+ self.w.indent()
+ for value in enum.value:
+ self.w.writelines("""
+ %(name)s = %(value)d,
+ """ % {
+ 'name': value.name,
+ 'value': value.number,
+ })
+ self.w.unindent()
+ self.w.writelines("""
+ };
+
+ """)
+
+ def write_field(self, field, ctor, ser, size, clear, methods):
+ """Write a proto field object to the generated file, including necessary
+ code in the constructor and serialization methods.
+ """
+ field_cpp_type = field_type_to_cpp_type(field)
+ repeated = field.label == FieldDescriptor.LABEL_REPEATED
+
+ element_cpp_type = field_cpp_type
+ if repeated:
+ field_cpp_type = 'std::vector< %s >' % field_cpp_type
+
+ member_name = field.name + '_'
+ element_name = member_name
+
+ # Data declaration
+ self.w.writelines("""
+ %(type)s %(member_name)s;
+ bool has_%(member_name)s;
+ """ % {
+ 'type': field_cpp_type,
+ 'member_name': member_name,
+ })
+
+ ctor.writelines("""
+ has_%(member_name)s = false;
+ """ % {
+ 'member_name': member_name,
+ })
+
+ methods.writelines("""
+ %(type)s* mutable_%(name)s() {
+ has_%(member_name)s = true;
+ return &%(member_name)s;
+ }
+ """ % {
+ 'name': field.name,
+ 'member_name': member_name,
+ 'type': field_cpp_type,
+ })
+
+ if repeated:
+ loop = """
+ for (%(type)s::const_iterator it_ = %(member_name)s.begin();
+ it_ != %(member_name)s.end(); it_++) {
+ """ % {
+ 'member_name': member_name,
+ 'type': field_cpp_type,
+ }
+
+ ser.writelines(loop)
+ ser.indent()
+
+ size.writelines(loop)
+ size.indent()
+
+ methods.writelines("""
+ void add_%(name)s(const %(type)s& value) {
+ has_%(member_name)s = true;
+ %(member_name)s.push_back(value);
+ }
+ """ % {
+ 'name': field.name,
+ 'member_name': member_name,
+ 'type': element_cpp_type,
+ })
+
+ element_name = '*it_'
+
+ if field.type == FieldDescriptor.TYPE_MESSAGE:
+ ser.writelines("""
+ if (has_%(member_name)s) {
+ WriteLengthDelimited(output__, %(number)s,
+ %(member_name)s.ByteSizeLong());
+ %(member_name)s.SerializeToOstream(output__);
+ }
+ """ % {
+ 'member_name': element_name,
+ 'number': field.number,
+ })
+
+ size.writelines("""
+ if (has_%(member_name)s) {
+ size += 1 + VarintSize32(%(member_name)s.ByteSizeLong());
+ size += %(member_name)s.ByteSizeLong();
+ }
+ """ % {
+ 'member_name': element_name,
+ })
+
+ clear.writelines("""
+ if (has_%(member_name)s) {
+ %(member_name)s.Clear();
+ has_%(member_name)s = false;
+ }
+ """ % {
+ 'member_name': member_name,
+ })
+ else:
+ (sizer, serializer, formatter) = ENCODING_MAP[field.type]
+ if formatter != None:
+ element_name = '%s(%s)' % (formatter, element_name)
+
+ ser.writelines("""
+ %(serializer)s(output__, %(field_number)s, %(element_name)s);
+ """ % {
+ 'serializer': serializer,
+ 'field_number': field.number,
+ 'element_name': element_name,
+ })
+
+ size.writelines("""
+ size += %(sizer)s(%(element_name)s) + 1;
+ """ % {
+ 'sizer': sizer,
+ 'element_name': element_name,
+ })
+
+ if repeated or field.type == FieldDescriptor.CPPTYPE_STRING:
+ clear.writelines("""
+ %(member_name)s.clear();
+ """ % {
+ 'member_name': member_name,
+ })
+ else:
+ reset = """
+ %(member_name)s = static_cast< %(type)s >(0);
+ """ % {
+ 'member_name': member_name,
+ 'type': field_cpp_type,
+ }
+ ctor.writelines(reset)
+ clear.writelines(reset)
+
+ methods.writelines("""
+ void set_%(name)s(const %(type)s& value) {
+ has_%(member_name)s = true;
+ %(member_name)s = value;
+ }
+ """ % {
+ 'name': field.name,
+ 'member_name': member_name,
+ 'type': field_cpp_type,
+ })
+
+ if repeated:
+ ser.unindent()
+ ser.writelines('}')
+ size.unindent()
+ size.writelines('}')
+
+ def func(self, f):
+ return self.w.stringwriter(prefix=f + ' {', suffix='}\n\n')
+
+ def write_message(self, message):
+ """Write a proto message object to the generated file, recursing into
+ nested messages, enums, and fields.
+ """
+ self.w.writelines("""
+ struct %(name)s {
+ """ % {
+ 'name': message.name,
+ })
+ self.w.indent()
+
+ # Constructor
+ ctor = self.func(message.name + '()')
+
+ # SerializeToOstream method
+ ser = self.func('void SerializeToOstream(std::ostream* output__) const')
+
+ size = self.func('size_t ByteSizeLong() const')
+ size.writelines("""
+ size_t size = 0;
+ """)
+
+ clear = self.func('void Clear()')
+
+ methods = self.w.stringwriter()
+
+ # Nested message type declarations
+ for nested in message.nested_type:
+ self.write_message(nested)
+
+ # Nested enum type declarations
+ for enum in message.enum_type:
+ self.write_enum(enum)
+
+ # Message fields
+ for field in message.field:
+ self.write_field(field, ctor, ser, size, clear, methods)
+ if len(message.field) > 0:
+ self.w.newline()
+
+ self.w.writelines(ctor.string())
+
+ # Disallow copy and assign constructors
+ self.w.writelines("""
+ %(name)s(const %(name)s&);
+ void operator=(const %(name)s&);
+
+ """ % {
+ 'name': message.name,
+ })
+
+ # SerializeToOstream method
+ self.w.writelines(ser.string())
+
+ # ByteSizeLong method
+ size.writelines('return size;')
+ self.w.writelines(size.string())
+
+ # Clear method
+ self.w.writelines(clear.string())
+
+ # Accessor methods
+ self.w.write(methods.string())
+
+ self.w.unindent()
+ self.w.writelines("""
+ };
+
+ """)
+
+ def write_proto(self, output_file, proto):
+ header_guard = 'NINJA_' + os.path.basename(output_file).upper()
+ header_guard = re.sub('[^a-zA-Z]', '_', header_guard)
+
+ self.w.writelines("""
+ // This file is autogenerated by %(generator)s, do not edit
+
+ #ifndef %(header_guard)s
+ #define %(header_guard)s
+
+ #include <inttypes.h>
+
+ #include <iostream>
+ #include <string>
+ #include <vector>
+
+ #include "proto.h"
+
+ namespace %(namespace)s {
+ """ % {
+ 'generator': os.path.basename(sys.argv[0]),
+ 'header_guard': header_guard,
+ 'namespace': proto.package,
+ })
+
+ for enum in proto.enum_type:
+ self.write_enum(enum)
+ for message in proto.message_type:
+ self.write_message(message)
+
+ self.w.writelines("""
+ }
+ #endif // %(header_guard)s
+ """ % {
+ 'header_guard': header_guard,
+ })
+
+class Writer:
+ """Class to write code to a generated file"""
+ def __init__(self, w, indent=0):
+ self.w = w
+ self.cur_indent = indent
+
+ def write(self, s):
+ self.w.write(s)
+
+ def writeln(self, s):
+ if len(s) > 0:
+ self.write(' '*self.cur_indent + s + '\n')
+ else:
+ self.newline()
+
+ def indent(self):
+ self.cur_indent = self.cur_indent + 2
+
+ def unindent(self):
+ self.cur_indent = self.cur_indent - 2
+
+ def newline(self):
+ self.write('\n')
+
+ def writelines(self, s):
+ lines = s.split('\n')
+ if len(lines) > 0:
+ if len(lines[0].strip()) == 0:
+ lines = lines[1:]
+ if len(lines) > 0:
+ first_indent = initial_indent(lines[0])
+
+ for line in lines[:-1]:
+ indent = min(initial_indent(line), first_indent)
+ self.writeln(line[indent:])
+
+ indent = min(initial_indent(lines[-1]), first_indent)
+ if lines[-1][indent:] != '':
+ self.writeln(lines[-1][indent:])
+
+ def stringwriter(self, prefix='', suffix=''):
+ """Returns an object with the same interface as Writer that buffers
+ its writes to be written out later.
+ """
+ return StringWriter(self.cur_indent, prefix, suffix)
+
+def initial_indent(s):
+ return len(s)-len(s.lstrip(' '))
+
+class StringWriter(Writer):
+ """Subclass of Writer that buffers its writes to be written out later."""
+ def __init__(self, indent, prefix, suffix):
+ self.buf = ''
+ self.prefix = prefix
+ self.suffix = suffix
+ self.cur_indent = indent
+ if self.prefix != '':
+ self.writelines(self.prefix)
+ self.indent()
+
+ def string(self):
+ if self.prefix != '':
+ self.unindent()
+ if self.suffix != '':
+ self.writelines(self.suffix)
+ return self.buf
+
+ def write(self, s):
+ self.buf += s
+
+def main():
+ if len(sys.argv) == 2 and sys.argv[1] == '--probe':
+ print('ok')
+ return
+
+ if len(sys.argv) != 3:
+ print('usage: %s <in> <out>' % sys.argv[0])
+ sys.exit(1)
+
+ input_file = sys.argv[1]
+ output_file = sys.argv[2]
+ tmp_output_file = output_file + '.tmp'
+
+ set = google.protobuf.descriptor_pb2.FileDescriptorSet()
+ try:
+ with open(input_file, 'rb') as f:
+ set.ParseFromString(f.read())
+ except IOError:
+ print('failed to read ' + input_file)
+ sys.exit(2)
+
+ if len(set.file) != 1:
+ print('expected exactly one file descriptor in ' + input_file)
+ print(set)
+ sys.exit(3)
+
+ proto = set.file[0]
+
+ with open(tmp_output_file, 'w') as out:
+ w = Generator(out)
+
+ w.write_proto(output_file, proto)
+
+ os.rename(tmp_output_file, output_file)
+
+if __name__ == '__main__':
+ main()
diff --git a/misc/generate_proto_header_test.py b/misc/generate_proto_header_test.py
new file mode 100755
index 0000000..318e803
--- /dev/null
+++ b/misc/generate_proto_header_test.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Write-only protobuf C++ code generator for a minimal runtime
+
+This script uses a descriptor proto generated by protoc and the descriptor_pb2
+distributed with python protobuf to iterate through the fields in a proto
+and write out simple C++ data objects with serialization methods. The generated
+files depend on a tiny runtime implemented in src/proto.h and src/proto.cc.
+"""
+
+from __future__ import print_function
+
+import unittest
+
+try:
+ from StringIO import StringIO
+except ImportError:
+ from io import StringIO
+
+import generate_proto_header
+
+class TestWriteLines(unittest.TestCase):
+ def setUp(self):
+ self.out = StringIO()
+ self.w = generate_proto_header.Writer(self.out)
+
+ def test_single_line(self):
+ self.w.writelines('abc')
+ self.assertEqual(self.out.getvalue(), 'abc\n')
+
+ def test_multi_line(self):
+ self.w.writelines("""
+ abc
+ def
+ """)
+ self.assertEqual(self.out.getvalue(), 'abc\ndef\n')
+
+ def test_multi_line_with_indent(self):
+ self.w.writelines("""
+ abc
+ def
+ """)
+ self.assertEqual(self.out.getvalue(), 'abc\n def\n')
+
+ def test_string_writer(self):
+ w = self.w.stringwriter()
+ w.writelines("""
+ abc
+ def
+ """)
+ self.w.writelines(w.string())
+ self.assertEqual(self.out.getvalue(), 'abc\n def\n')
+
+ def test_string_writer_prefix(self):
+ w = self.w.stringwriter(
+ prefix="""
+ abc
+ def
+ """,
+ suffix="""
+ lmnop
+ """)
+ w.writelines("""
+ ghi
+ jk
+ """)
+ self.w.writelines(w.string())
+ self.assertEqual(self.out.getvalue(),
+ 'abc\n def\n ghi\n jk\nlmnop\n')
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/src/build.cc b/src/build.cc
index a055738..9fce38e 100644
--- a/src/build.cc
+++ b/src/build.cc
@@ -36,7 +36,9 @@
#include "deps_log.h"
#include "disk_interface.h"
#include "graph.h"
+#include "metrics.h"
#include "state.h"
+#include "status.h"
#include "subprocess.h"
#include "util.h"
@@ -76,232 +78,6 @@
} // namespace
-BuildStatus::BuildStatus(const BuildConfig& config)
- : config_(config),
- start_time_millis_(GetTimeMillis()),
- started_edges_(0), finished_edges_(0), total_edges_(0),
- progress_status_format_(NULL),
- overall_rate_(), current_rate_(config.parallelism) {
-
- // Don't do anything fancy in verbose mode.
- if (config_.verbosity != BuildConfig::NORMAL)
- printer_.set_smart_terminal(false);
-
- progress_status_format_ = getenv("NINJA_STATUS");
- if (!progress_status_format_)
- progress_status_format_ = "[%f/%t] ";
-}
-
-void BuildStatus::PlanHasTotalEdges(int total) {
- total_edges_ = total;
-}
-
-void BuildStatus::BuildEdgeStarted(Edge* edge) {
- assert(running_edges_.find(edge) == running_edges_.end());
- int start_time = (int)(GetTimeMillis() - start_time_millis_);
- running_edges_.insert(make_pair(edge, start_time));
- ++started_edges_;
-
- if (edge->use_console() || printer_.is_smart_terminal())
- PrintStatus(edge, kEdgeStarted);
-
- if (edge->use_console())
- printer_.SetConsoleLocked(true);
-}
-
-void BuildStatus::BuildEdgeFinished(Edge* edge,
- bool success,
- const string& output,
- int* start_time,
- int* end_time) {
- int64_t now = GetTimeMillis();
-
- ++finished_edges_;
-
- RunningEdgeMap::iterator i = running_edges_.find(edge);
- *start_time = i->second;
- *end_time = (int)(now - start_time_millis_);
- running_edges_.erase(i);
-
- if (edge->use_console())
- printer_.SetConsoleLocked(false);
-
- if (config_.verbosity == BuildConfig::QUIET)
- return;
-
- if (!edge->use_console())
- PrintStatus(edge, kEdgeFinished);
-
- // Print the command that is spewing before printing its output.
- if (!success) {
- string outputs;
- for (vector<Node*>::const_iterator o = edge->outputs_.begin();
- o != edge->outputs_.end(); ++o)
- outputs += (*o)->path() + " ";
-
- printer_.PrintOnNewLine("FAILED: " + outputs + "\n");
- printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n");
- }
-
- if (!output.empty()) {
- // ninja sets stdout and stderr of subprocesses to a pipe, to be able to
- // check if the output is empty. Some compilers, e.g. clang, check
- // isatty(stderr) to decide if they should print colored output.
- // To make it possible to use colored output with ninja, subprocesses should
- // be run with a flag that forces them to always print color escape codes.
- // To make sure these escape codes don't show up in a file if ninja's output
- // is piped to a file, ninja strips ansi escape codes again if it's not
- // writing to a |smart_terminal_|.
- // (Launching subprocesses in pseudo ttys doesn't work because there are
- // only a few hundred available on some systems, and ninja can launch
- // thousands of parallel compile commands.)
- string final_output;
- if (!printer_.supports_color())
- final_output = StripAnsiEscapeCodes(output);
- else
- final_output = output;
-
-#ifdef _WIN32
- // Fix extra CR being added on Windows, writing out CR CR LF (#773)
- _setmode(_fileno(stdout), _O_BINARY); // Begin Windows extra CR fix
-#endif
-
- printer_.PrintOnNewLine(final_output);
-
-#ifdef _WIN32
- _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix
-#endif
- }
-}
-
-void BuildStatus::BuildLoadDyndeps() {
- // The DependencyScan calls EXPLAIN() to print lines explaining why
- // it considers a portion of the graph to be out of date. Normally
- // this is done before the build starts, but our caller is about to
- // load a dyndep file during the build. Doing so may generate more
- // exlanation lines (via fprintf directly to stderr), but in an
- // interactive console the cursor is currently at the end of a status
- // line. Start a new line so that the first explanation does not
- // append to the status line. After the explanations are done a
- // new build status line will appear.
- if (g_explaining)
- printer_.PrintOnNewLine("");
-}
-
-void BuildStatus::BuildStarted() {
- overall_rate_.Restart();
- current_rate_.Restart();
-}
-
-void BuildStatus::BuildFinished() {
- printer_.SetConsoleLocked(false);
- printer_.PrintOnNewLine("");
-}
-
-string BuildStatus::FormatProgressStatus(
- const char* progress_status_format, EdgeStatus status) const {
- string out;
- char buf[32];
- int percent;
- for (const char* s = progress_status_format; *s != '\0'; ++s) {
- if (*s == '%') {
- ++s;
- switch (*s) {
- case '%':
- out.push_back('%');
- break;
-
- // Started edges.
- case 's':
- snprintf(buf, sizeof(buf), "%d", started_edges_);
- out += buf;
- break;
-
- // Total edges.
- case 't':
- snprintf(buf, sizeof(buf), "%d", total_edges_);
- out += buf;
- break;
-
- // Running edges.
- case 'r': {
- int running_edges = started_edges_ - finished_edges_;
- // count the edge that just finished as a running edge
- if (status == kEdgeFinished)
- running_edges++;
- snprintf(buf, sizeof(buf), "%d", running_edges);
- out += buf;
- break;
- }
-
- // Unstarted edges.
- case 'u':
- snprintf(buf, sizeof(buf), "%d", total_edges_ - started_edges_);
- out += buf;
- break;
-
- // Finished edges.
- case 'f':
- snprintf(buf, sizeof(buf), "%d", finished_edges_);
- out += buf;
- break;
-
- // Overall finished edges per second.
- case 'o':
- overall_rate_.UpdateRate(finished_edges_);
- SnprintfRate(overall_rate_.rate(), buf, "%.1f");
- out += buf;
- break;
-
- // Current rate, average over the last '-j' jobs.
- case 'c':
- current_rate_.UpdateRate(finished_edges_);
- SnprintfRate(current_rate_.rate(), buf, "%.1f");
- out += buf;
- break;
-
- // Percentage
- case 'p':
- percent = (100 * finished_edges_) / total_edges_;
- snprintf(buf, sizeof(buf), "%3i%%", percent);
- out += buf;
- break;
-
- case 'e': {
- double elapsed = overall_rate_.Elapsed();
- snprintf(buf, sizeof(buf), "%.3f", elapsed);
- out += buf;
- break;
- }
-
- default:
- Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s);
- return "";
- }
- } else {
- out.push_back(*s);
- }
- }
-
- return out;
-}
-
-void BuildStatus::PrintStatus(Edge* edge, EdgeStatus status) {
- if (config_.verbosity == BuildConfig::QUIET)
- return;
-
- bool force_full_command = config_.verbosity == BuildConfig::VERBOSE;
-
- string to_print = edge->GetBinding("description");
- if (to_print.empty() || force_full_command)
- to_print = edge->GetBinding("command");
-
- to_print = FormatProgressStatus(progress_status_format_, status) + to_print;
-
- printer_.Print(to_print,
- force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE);
-}
-
Plan::Plan(Builder* builder)
: builder_(builder)
, command_edges_(0)
@@ -378,7 +154,7 @@
Edge* Plan::FindWork() {
if (ready_.empty())
return NULL;
- set<Edge*>::iterator e = ready_.begin();
+ EdgeSet::iterator e = ready_.begin();
Edge* edge = *e;
ready_.erase(e);
return edge;
@@ -444,8 +220,9 @@
}
// See if we we want any edges from this node.
- for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
- oe != node->out_edges().end(); ++oe) {
+ const std::vector<Edge*> out_edges = node->GetOutEdges();
+ for (vector<Edge*>::const_iterator oe = out_edges.begin();
+ oe != out_edges.end(); ++oe) {
map<Edge*, Want>::iterator want_e = want_.find(*oe);
if (want_e == want_.end())
continue;
@@ -475,8 +252,9 @@
bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) {
node->set_dirty(false);
- for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
- oe != node->out_edges().end(); ++oe) {
+ const std::vector<Edge*> out_edges = node->GetOutEdges();
+ for (vector<Edge*>::const_iterator oe = out_edges.begin();
+ oe != out_edges.end(); ++oe) {
// Don't process edges that we don't actually want.
map<Edge*, Want>::iterator want_e = want_.find(*oe);
if (want_e == want_.end() || want_e->second == kWantNothing)
@@ -486,6 +264,10 @@
if ((*oe)->deps_missing_)
continue;
+ // No need to clean a phony output edge, as it's always dirty
+ if ((*oe)->IsPhonyOutput())
+ continue;
+
// If all non-order-only inputs for this edge are now clean,
// we might have changed the dirty state of the outputs.
vector<Node*>::iterator
@@ -576,8 +358,9 @@
// Add out edges from this node that are in the plan (just as
// Plan::NodeFinished would have without taking the dyndep code path).
- for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
- oe != node->out_edges().end(); ++oe) {
+ const std::vector<Edge*> out_edges = node->GetOutEdges();
+ for (vector<Edge*>::const_iterator oe = out_edges.begin();
+ oe != out_edges.end(); ++oe) {
map<Edge*, Want>::iterator want_e = want_.find(*oe);
if (want_e == want_.end())
continue;
@@ -611,8 +394,21 @@
Node* n = *i;
// Check if this dependent node is now dirty. Also checks for new cycles.
- if (!scan->RecomputeDirty(n, err))
+ std::vector<Node*> validation_nodes;
+ if (!scan->RecomputeDirty(n, &validation_nodes, err))
return false;
+
+ // Add any validation nodes found during RecomputeDirty as new top level
+ // targets.
+ for (std::vector<Node*>::iterator v = validation_nodes.begin();
+ v != validation_nodes.end(); ++v) {
+ if (Edge* in_edge = (*v)->in_edge()) {
+ if (!in_edge->outputs_ready() &&
+ !AddTarget(*v, err)) {
+ return false;
+ }
+ }
+ }
if (!n->dirty())
continue;
@@ -632,8 +428,9 @@
}
void Plan::UnmarkDependents(Node* node, set<Node*>* dependents) {
- for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
- oe != node->out_edges().end(); ++oe) {
+ const std::vector<Edge*> out_edges = node->GetOutEdges();
+ for (vector<Edge*>::const_iterator oe = out_edges.begin();
+ oe != out_edges.end(); ++oe) {
Edge* edge = *oe;
map<Edge*, Want>::iterator want_e = want_.find(edge);
@@ -714,6 +511,9 @@
}
result->status = subproc->Finish();
+#ifndef _WIN32
+ result->rusage = *subproc->GetUsage();
+#endif
result->output = subproc->GetOutput();
map<Subprocess*, Edge*>::iterator e = subproc_to_edge_.find(subproc);
@@ -726,12 +526,12 @@
Builder::Builder(State* state, const BuildConfig& config,
BuildLog* build_log, DepsLog* deps_log,
- DiskInterface* disk_interface)
- : state_(state), config_(config),
- plan_(this), disk_interface_(disk_interface),
+ DiskInterface* disk_interface, Status* status,
+ int64_t start_time_millis)
+ : state_(state), config_(config), plan_(this), status_(status),
+ start_time_millis_(start_time_millis), disk_interface_(disk_interface),
scan_(state, build_log, deps_log, disk_interface,
- &config_.depfile_parser_options) {
- status_ = new BuildStatus(config);
+ &config_.depfile_parser_options, config.uses_phony_outputs) {
}
Builder::~Builder() {
@@ -745,6 +545,8 @@
for (vector<Edge*>::iterator e = active_edges.begin();
e != active_edges.end(); ++e) {
+ if ((*e)->IsPhonyOutput())
+ continue;
string depfile = (*e)->GetUnescapedDepfile();
for (vector<Node*>::iterator o = (*e)->outputs_.begin();
o != (*e)->outputs_.end(); ++o) {
@@ -756,10 +558,11 @@
// mentioned in a depfile, and the command touches its depfile
// but is interrupted before it touches its output file.)
string err;
- TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), &err);
- if (new_mtime == -1) // Log and ignore Stat() errors.
- Error("%s", err.c_str());
- if (!depfile.empty() || (*o)->mtime() != new_mtime)
+ bool is_dir = false;
+ TimeStamp new_mtime = disk_interface_->LStat((*o)->path(), &is_dir, &err);
+ if (new_mtime == -1) // Log and ignore LStat() errors.
+ status_->Error("%s", err.c_str());
+ if (!is_dir && (!depfile.empty() || (*o)->mtime() != new_mtime))
disk_interface_->RemoveFile((*o)->path());
}
if (!depfile.empty())
@@ -774,22 +577,41 @@
*err = "unknown target: '" + name + "'";
return NULL;
}
- if (!AddTarget(node, err))
+ if (!AddTargets({ node }, err))
return NULL;
return node;
}
-bool Builder::AddTarget(Node* node, string* err) {
- if (!scan_.RecomputeDirty(node, err))
+bool Builder::AddTargets(const std::vector<Node*> &nodes, string* err) {
+ std::vector<Node*> validation_nodes;
+ if (!scan_.RecomputeNodesDirty(nodes, &validation_nodes, err))
return false;
- if (Edge* in_edge = node->in_edge()) {
- if (in_edge->outputs_ready())
- return true; // Nothing to do.
+ for (Node* node : nodes) {
+ std::string plan_err;
+ if (!plan_.AddTarget(node, &plan_err)) {
+ if (!plan_err.empty()) {
+ *err = plan_err;
+ return false;
+ } else {
+ // Added a target that is already up-to-date; not really
+ // an error.
+ }
+ }
}
- if (!plan_.AddTarget(node, err))
- return false;
+ for (Node* node : validation_nodes) {
+ std::string plan_err;
+ if (!plan_.AddTarget(node, &plan_err)) {
+ if (!plan_err.empty()) {
+ *err = plan_err;
+ return false;
+ } else {
+ // Added a target that is already up-to-date; not really
+ // an error.
+ }
+ }
+ }
return true;
}
@@ -897,14 +719,29 @@
if (edge->is_phony())
return true;
- status_->BuildEdgeStarted(edge);
+ int64_t start_time_millis = GetTimeMillis() - start_time_millis_;
+ running_edges_.insert(make_pair(edge, start_time_millis));
- // Create directories necessary for outputs.
- // XXX: this will block; do we care?
- for (vector<Node*>::iterator o = edge->outputs_.begin();
- o != edge->outputs_.end(); ++o) {
- if (!disk_interface_->MakeDirs((*o)->path()))
- return false;
+ status_->BuildEdgeStarted(edge, start_time_millis);
+
+ if (!edge->IsPhonyOutput()) {
+ for (vector<Node*>::iterator o = edge->outputs_.begin();
+ o != edge->outputs_.end(); ++o) {
+ // Create directories necessary for outputs.
+ // XXX: this will block; do we care?
+ if (!disk_interface_->MakeDirs((*o)->path()))
+ return false;
+
+ if (!(*o)->exists())
+ continue;
+
+ // Remove existing outputs for non-restat rules.
+ // XXX: this will block; do we care?
+ if (config_.pre_remove_output_files && !edge->IsRestat() && !config_.dry_run) {
+ if (disk_interface_->RemoveFile((*o)->path()) < 0)
+ return false;
+ }
+ }
}
// Create response file, if needed
@@ -929,71 +766,115 @@
METRIC_RECORD("FinishCommand");
Edge* edge = result->edge;
+ bool phony_output = edge->IsPhonyOutput();
- // First try to extract dependencies from the result, if any.
- // This must happen first as it filters the command output (we want
- // to filter /showIncludes output, even on compile failure) and
- // extraction itself can fail, which makes the command fail from a
- // build perspective.
vector<Node*> deps_nodes;
string deps_type = edge->GetBinding("deps");
- const string deps_prefix = edge->GetBinding("msvc_deps_prefix");
- if (!deps_type.empty()) {
- string extract_err;
- if (!ExtractDeps(result, deps_type, deps_prefix, &deps_nodes,
- &extract_err) &&
- result->success()) {
- if (!result->output.empty())
- result->output.append("\n");
- result->output.append(extract_err);
- result->status = ExitFailure;
+ if (!phony_output) {
+ // First try to extract dependencies from the result, if any.
+ // This must happen first as it filters the command output (we want
+ // to filter /showIncludes output, even on compile failure) and
+ // extraction itself can fail, which makes the command fail from a
+ // build perspective.
+ const string deps_prefix = edge->GetBinding("msvc_deps_prefix");
+ if (!deps_type.empty()) {
+ string extract_err;
+ if (!ExtractDeps(result, deps_type, deps_prefix, &deps_nodes,
+ &extract_err) &&
+ result->success()) {
+ if (!result->output.empty())
+ result->output.append("\n");
+ result->output.append(extract_err);
+ result->status = ExitFailure;
+ }
}
}
- int start_time, end_time;
- status_->BuildEdgeFinished(edge, result->success(), result->output,
- &start_time, &end_time);
-
- // The rest of this function only applies to successful commands.
- if (!result->success()) {
- return plan_.EdgeFinished(edge, Plan::kEdgeFailed, err);
- }
+ int64_t start_time_millis, end_time_millis;
+ RunningEdgeMap::iterator i = running_edges_.find(edge);
+ start_time_millis = i->second;
+ end_time_millis = GetTimeMillis() - start_time_millis_;
+ running_edges_.erase(i);
// Restat the edge outputs
TimeStamp output_mtime = 0;
- bool restat = edge->GetBindingBool("restat");
- if (!config_.dry_run) {
- bool node_cleaned = false;
+ if (result->success() && !config_.dry_run && !phony_output) {
+ bool restat = edge->IsRestat();
+ vector<Node*> nodes_cleaned;
+
+ TimeStamp newest_input = 0;
+ Node* newest_input_node = nullptr;
+ for (vector<Node*>::iterator i = edge->inputs_.begin();
+ i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
+ TimeStamp input_mtime = (*i)->mtime();
+ if (input_mtime == -1)
+ return false;
+ if (input_mtime > newest_input) {
+ newest_input = input_mtime;
+ newest_input_node = (*i);
+ }
+ }
for (vector<Node*>::iterator o = edge->outputs_.begin();
o != edge->outputs_.end(); ++o) {
- TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err);
- if (new_mtime == -1)
+ bool is_dir = false;
+ TimeStamp old_mtime = (*o)->mtime();
+ if (!(*o)->LStat(disk_interface_, &is_dir, err))
return false;
+ TimeStamp new_mtime = (*o)->mtime();
+ if (config_.uses_phony_outputs) {
+ if (new_mtime == 0) {
+ if (!result->output.empty())
+ result->output.append("\n");
+ result->output.append("ninja: output file missing after successful execution: ");
+ result->output.append((*o)->path());
+ if (config_.missing_output_file_should_err) {
+ result->status = ExitFailure;
+ }
+ } else if (!restat && new_mtime < newest_input) {
+ if (!result->output.empty())
+ result->output.append("\n");
+ result->output.append("ninja: Missing `restat`? An output file is older than the most recent input:\n output: ");
+ result->output.append((*o)->path());
+ result->output.append("\n input: ");
+ result->output.append(newest_input_node->path());
+ if (config_.old_output_should_err) {
+ result->status = ExitFailure;
+ }
+ }
+ if (is_dir) {
+ if (!result->output.empty())
+ result->output.append("\n");
+ result->output.append("ninja: outputs should be files, not directories: ");
+ result->output.append((*o)->path());
+ if (config_.output_directory_should_err) {
+ result->status = ExitFailure;
+ }
+ }
+ }
if (new_mtime > output_mtime)
output_mtime = new_mtime;
- if ((*o)->mtime() == new_mtime && restat) {
+ if (old_mtime == new_mtime && restat) {
+ nodes_cleaned.push_back(*o);
+ continue;
+ }
+ }
+
+ status_->BuildEdgeFinished(edge, end_time_millis, result);
+
+ if (result->success() && !nodes_cleaned.empty()) {
+ for (vector<Node*>::iterator o = nodes_cleaned.begin();
+ o != nodes_cleaned.end(); ++o) {
// The rule command did not change the output. Propagate the clean
// state through the build graph.
// Note that this also applies to nonexistent outputs (mtime == 0).
if (!plan_.CleanNode(&scan_, *o, err))
return false;
- node_cleaned = true;
}
- }
- if (node_cleaned) {
- TimeStamp restat_mtime = 0;
// If any output was cleaned, find the most recent mtime of any
// (existing) non-order-only input or the depfile.
- for (vector<Node*>::iterator i = edge->inputs_.begin();
- i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
- TimeStamp input_mtime = disk_interface_->Stat((*i)->path(), err);
- if (input_mtime == -1)
- return false;
- if (input_mtime > restat_mtime)
- restat_mtime = input_mtime;
- }
+ TimeStamp restat_mtime = newest_input;
string depfile = edge->GetUnescapedDepfile();
if (restat_mtime != 0 && deps_type.empty() && !depfile.empty()) {
@@ -1010,28 +891,34 @@
output_mtime = restat_mtime;
}
+ } else {
+ status_->BuildEdgeFinished(edge, end_time_millis, result);
}
- if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err))
+ if (!plan_.EdgeFinished(edge, result->success() ? Plan::kEdgeSucceeded : Plan::kEdgeFailed, err))
return false;
+ // The rest of this function only applies to successful commands.
+ if (!result->success()) {
+ return true;
+ }
+
// Delete any left over response file.
string rspfile = edge->GetUnescapedRspfile();
if (!rspfile.empty() && !g_keep_rsp)
disk_interface_->RemoveFile(rspfile);
- if (scan_.build_log()) {
- if (!scan_.build_log()->RecordCommand(edge, start_time, end_time,
- output_mtime)) {
+ if (scan_.build_log() && !phony_output) {
+ if (!scan_.build_log()->RecordCommand(edge, start_time_millis,
+ end_time_millis, output_mtime)) {
*err = string("Error writing to build log: ") + strerror(errno);
return false;
}
}
- if (!deps_type.empty() && !config_.dry_run) {
- assert(edge->outputs_.size() == 1 && "should have been rejected by parser");
+ if (!deps_type.empty() && !config_.dry_run && !phony_output) {
Node* out = edge->outputs_[0];
- TimeStamp deps_mtime = disk_interface_->Stat(out->path(), err);
+ TimeStamp deps_mtime = disk_interface_->LStat(out->path(), nullptr, err);
if (deps_mtime == -1)
return false;
if (!scan_.deps_log()->RecordDeps(out, deps_mtime, deps_nodes)) {
@@ -1076,6 +963,15 @@
break;
case DiskInterface::NotFound:
err->clear();
+ // We only care if the depfile is missing when the tool succeeded.
+ if (!config_.dry_run && result->status == ExitSuccess) {
+ if (config_.missing_depfile_should_err) {
+ *err = string("depfile is missing");
+ return false;
+ } else {
+ status_->Warning("depfile is missing (%s for %s)", depfile.c_str(), result->edge->outputs_[0]->path().c_str());
+ }
+ }
break;
case DiskInterface::OtherError:
return false;
diff --git a/src/build.h b/src/build.h
index ab59f0c..61d3f78 100644
--- a/src/build.h
+++ b/src/build.h
@@ -19,24 +19,25 @@
#include <map>
#include <memory>
#include <queue>
-#include <set>
#include <string>
#include <vector>
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
#include "depfile_parser.h"
#include "graph.h" // XXX needed for DependencyScan; should rearrange.
#include "exit_status.h"
-#include "line_printer.h"
-#include "metrics.h"
#include "util.h" // int64_t
struct BuildLog;
-struct BuildStatus;
struct Builder;
struct DiskInterface;
struct Edge;
struct Node;
struct State;
+struct Status;
/// Plan stores the state of a build plan: what we intend to build,
/// which steps we're ready to execute.
@@ -122,7 +123,7 @@
/// we want for the edge.
map<Edge*, Want> want_;
- set<Edge*> ready_;
+ EdgeSet ready_;
Builder* builder_;
@@ -146,6 +147,9 @@
Result() : edge(NULL) {}
Edge* edge;
ExitStatus status;
+#ifndef _WIN32
+ struct rusage rusage;
+#endif
string output;
bool success() const { return status == ExitSuccess; }
};
@@ -159,7 +163,14 @@
/// Options (e.g. verbosity, parallelism) passed to a build.
struct BuildConfig {
BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1),
- failures_allowed(1), max_load_average(-0.0f) {}
+ failures_allowed(1), max_load_average(-0.0f),
+ frontend(NULL), frontend_file(NULL),
+ missing_depfile_should_err(false),
+ uses_phony_outputs(false),
+ output_directory_should_err(false),
+ missing_output_file_should_err(false),
+ old_output_should_err(false),
+ pre_remove_output_files(false) {}
enum Verbosity {
NORMAL,
@@ -174,23 +185,51 @@
/// means that we do not have any limit.
double max_load_average;
DepfileParserOptions depfile_parser_options;
+
+ /// Command to execute to handle build output
+ const char* frontend;
+
+ /// File to write build output to
+ const char* frontend_file;
+
+ /// Whether a missing depfile should warn or print an error.
+ bool missing_depfile_should_err;
+
+ /// Whether the generator uses 'phony_output's
+ /// Controls the warnings below
+ bool uses_phony_outputs;
+
+ /// Whether an output can be a directory
+ bool output_directory_should_err;
+
+ /// Whether a missing output file should warn or print an error.
+ bool missing_output_file_should_err;
+
+ /// Whether an output with an older timestamp than the inputs should
+ /// warn or print an error.
+ bool old_output_should_err;
+
+ /// Whether to remove outputs before executing rule commands
+ bool pre_remove_output_files;
};
/// Builder wraps the build process: starting commands, updating status.
struct Builder {
Builder(State* state, const BuildConfig& config,
BuildLog* build_log, DepsLog* deps_log,
- DiskInterface* disk_interface);
+ DiskInterface* disk_interface, Status* status,
+ int64_t start_time_millis);
~Builder();
/// Clean up after interrupted commands by deleting output files.
void Cleanup();
+ /// Used by tests.
Node* AddTarget(const string& name, string* err);
- /// Add a target to the build, scanning dependencies.
+ /// Add targets to the build, scanning dependencies.
/// @return false on error.
- bool AddTarget(Node* target, string* err);
+ bool AddTargets(const std::vector<Node*>& targets, string* err);
/// Returns true if the build targets are already up to date.
bool AlreadyUpToDate() const;
@@ -221,13 +260,20 @@
#else
unique_ptr<CommandRunner> command_runner_; // auto_ptr was removed in C++17.
#endif
- BuildStatus* status_;
+ Status* status_;
private:
bool ExtractDeps(CommandRunner::Result* result, const string& deps_type,
const string& deps_prefix, vector<Node*>* deps_nodes,
string* err);
+ /// Map of running edge to time the edge started running.
+ typedef map<Edge*, int> RunningEdgeMap;
+ RunningEdgeMap running_edges_;
+
+ /// Time the build started.
+ int64_t start_time_millis_;
+
DiskInterface* disk_interface_;
DependencyScan scan_;
@@ -236,103 +282,4 @@
void operator=(const Builder &other); // DO NOT IMPLEMENT
};
-/// Tracks the status of a build: completion fraction, printing updates.
-struct BuildStatus {
- explicit BuildStatus(const BuildConfig& config);
- void PlanHasTotalEdges(int total);
- void BuildEdgeStarted(Edge* edge);
- void BuildEdgeFinished(Edge* edge, bool success, const string& output,
- int* start_time, int* end_time);
- void BuildLoadDyndeps();
- void BuildStarted();
- void BuildFinished();
-
- enum EdgeStatus {
- kEdgeStarted,
- kEdgeFinished,
- };
-
- /// Format the progress status string by replacing the placeholders.
- /// See the user manual for more information about the available
- /// placeholders.
- /// @param progress_status_format The format of the progress status.
- /// @param status The status of the edge.
- string FormatProgressStatus(const char* progress_status_format,
- EdgeStatus status) const;
-
- private:
- void PrintStatus(Edge* edge, EdgeStatus status);
-
- const BuildConfig& config_;
-
- /// Time the build started.
- int64_t start_time_millis_;
-
- int started_edges_, finished_edges_, total_edges_;
-
- /// Map of running edge to time the edge started running.
- typedef map<Edge*, int> RunningEdgeMap;
- RunningEdgeMap running_edges_;
-
- /// Prints progress output.
- LinePrinter printer_;
-
- /// The custom progress status format to use.
- const char* progress_status_format_;
-
- template<size_t S>
- void SnprintfRate(double rate, char(&buf)[S], const char* format) const {
- if (rate == -1)
- snprintf(buf, S, "?");
- else
- snprintf(buf, S, format, rate);
- }
-
- struct RateInfo {
- RateInfo() : rate_(-1) {}
-
- void Restart() { stopwatch_.Restart(); }
- double Elapsed() const { return stopwatch_.Elapsed(); }
- double rate() { return rate_; }
-
- void UpdateRate(int edges) {
- if (edges && stopwatch_.Elapsed())
- rate_ = edges / stopwatch_.Elapsed();
- }
-
- private:
- double rate_;
- Stopwatch stopwatch_;
- };
-
- struct SlidingRateInfo {
- SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {}
-
- void Restart() { stopwatch_.Restart(); }
- double rate() { return rate_; }
-
- void UpdateRate(int update_hint) {
- if (update_hint == last_update_)
- return;
- last_update_ = update_hint;
-
- if (times_.size() == N)
- times_.pop();
- times_.push(stopwatch_.Elapsed());
- if (times_.back() != times_.front())
- rate_ = times_.size() / (times_.back() - times_.front());
- }
-
- private:
- double rate_;
- Stopwatch stopwatch_;
- const size_t N;
- queue<double> times_;
- int last_update_;
- };
-
- mutable RateInfo overall_rate_;
- mutable SlidingRateInfo current_rate_;
-};
-
#endif // NINJA_BUILD_H_
diff --git a/src/build_log.cc b/src/build_log.cc
index c4a08a0..0ebc45d 100644
--- a/src/build_log.cc
+++ b/src/build_log.cc
@@ -22,6 +22,7 @@
#include "build_log.h"
+#include <assert.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
@@ -31,9 +32,13 @@
#include <unistd.h>
#endif
+#include <numeric>
+
#include "build.h"
+#include "disk_interface.h"
#include "graph.h"
#include "metrics.h"
+#include "parallel_map.h"
#include "util.h"
#if defined(_MSC_VER) && (_MSC_VER < 1800)
#define strtoll _strtoi64
@@ -49,9 +54,11 @@
namespace {
const char kFileSignature[] = "# ninja log v%d\n";
-const int kOldestSupportedVersion = 4;
+const int kOldestSupportedVersion = 5;
const int kCurrentVersion = 5;
+const char kFieldSeparator = '\t';
+
// 64bit MurmurHash2, by Austin Appleby
#if defined(_MSC_VER)
#define BIG_CONSTANT(x) (x)
@@ -105,13 +112,14 @@
// static
uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) {
- return MurmurHash64A(command.str_, command.len_);
+ METRIC_RECORD("hash command");
+ return MurmurHash64A(command.data(), command.size());
}
-BuildLog::LogEntry::LogEntry(const string& output)
+BuildLog::LogEntry::LogEntry(const HashedStrView& output)
: output(output) {}
-BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash,
+BuildLog::LogEntry::LogEntry(const HashedStrView& output, uint64_t command_hash,
int start_time, int end_time, TimeStamp restat_mtime)
: output(output), command_hash(command_hash),
start_time(start_time), end_time(end_time), mtime(restat_mtime)
@@ -157,13 +165,11 @@
TimeStamp mtime) {
string command = edge->EvaluateCommand(true);
uint64_t command_hash = LogEntry::HashCommand(command);
- for (vector<Node*>::iterator out = edge->outputs_.begin();
- out != edge->outputs_.end(); ++out) {
- const string& path = (*out)->path();
- Entries::iterator i = entries_.find(path);
+ for (Node* out : edge->outputs_) {
+ const HashedStr& path = out->path_hashed();
LogEntry* log_entry;
- if (i != entries_.end()) {
- log_entry = i->second;
+ if (LogEntry** i = entries_.Lookup(path)) {
+ log_entry = *i;
} else {
log_entry = new LogEntry(path);
entries_.insert(Entries::value_type(log_entry->output, log_entry));
@@ -190,164 +196,276 @@
log_file_ = NULL;
}
-struct LineReader {
- explicit LineReader(FILE* file)
- : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {
- memset(buf_, 0, sizeof(buf_));
+/// Retrieve the next tab-delimited or LF-delimited piece in the input.
+///
+/// Specifically, the function searches for a separator, starting at the given
+/// input offset. If the function finds the separator, it removes everything up
+/// to and including the separator from |*input|, places it in |*out|, and
+/// returns true. Otherwise, it zeroes |*out| and returns false.
+static bool GetNextPiece(StringPiece* input, char sep, StringPiece* out,
+ size_t start_offset=0) {
+ assert(input != out);
+ const char* data = input->data();
+ const char* split = nullptr;
+
+ // memchr(NULL, NULL, 0) has undefined behavior, so avoid calling memchr when
+ // input->data() is nullptr. If input->size() is non-zero, its data() will be
+ // non-null.
+ if (start_offset < input->size()) {
+ split = static_cast<const char*>(
+ memchr(data + start_offset, sep, input->size() - start_offset));
}
- // Reads a \n-terminated line from the file passed to the constructor.
- // On return, *line_start points to the beginning of the next line, and
- // *line_end points to the \n at the end of the line. If no newline is seen
- // in a fixed buffer size, *line_end is set to NULL. Returns false on EOF.
- bool ReadLine(char** line_start, char** line_end) {
- if (line_start_ >= buf_end_ || !line_end_) {
- // Buffer empty, refill.
- size_t size_read = fread(buf_, 1, sizeof(buf_), file_);
- if (!size_read)
- return false;
- line_start_ = buf_;
- buf_end_ = buf_ + size_read;
- } else {
- // Advance to next line in buffer.
- line_start_ = line_end_ + 1;
- }
-
- line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
- if (!line_end_) {
- // No newline. Move rest of data to start of buffer, fill rest.
- size_t already_consumed = line_start_ - buf_;
- size_t size_rest = (buf_end_ - buf_) - already_consumed;
- memmove(buf_, line_start_, size_rest);
-
- size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_);
- buf_end_ = buf_ + size_rest + read;
- line_start_ = buf_;
- line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
- }
-
- *line_start = line_start_;
- *line_end = line_end_;
+ if (split != nullptr) {
+ size_t len = split + 1 - data;
+ *out = input->substr(0, len);
+ *input = input->substr(len);
return true;
+ } else {
+ *out = {};
+ return false;
}
+}
- private:
- FILE* file_;
- char buf_[256 << 10];
- char* buf_end_; // Points one past the last valid byte in |buf_|.
+struct BuildLogInput {
+ std::unique_ptr<LoadedFile> file;
+ int log_version = 0;
- char* line_start_;
- // Points at the next \n in buf_ after line_start, or NULL.
- char* line_end_;
+ // Content excluding the file header.
+ StringPiece content;
};
-bool BuildLog::Load(const string& path, string* err) {
- METRIC_RECORD(".ninja_log load");
- FILE* file = fopen(path.c_str(), "r");
- if (!file) {
- if (errno == ENOENT)
- return true;
- *err = strerror(errno);
+static bool OpenBuildLogForReading(const std::string& path,
+ BuildLogInput* log,
+ std::string* err) {
+ *log = {};
+
+ RealDiskInterface file_reader;
+ std::string load_err;
+ switch (file_reader.LoadFile(path, &log->file, &load_err)) {
+ case FileReader::Okay:
+ break;
+ case FileReader::NotFound:
+ return true;
+ default:
+ *err = load_err;
return false;
}
- int log_version = 0;
- int unique_entry_count = 0;
- int total_entry_count = 0;
+ // We need a NUL terminator after the log file's content so that we can call
+ // atoi/atol/strtoull with a pointer to within the content.
+ log->content = log->file->content_with_nul();
+ log->content.remove_suffix(1);
- LineReader reader(file);
- char* line_start = 0;
- char* line_end = 0;
- while (reader.ReadLine(&line_start, &line_end)) {
- if (!log_version) {
- sscanf(line_start, kFileSignature, &log_version);
+ StringPiece header_line;
+ if (GetNextPiece(&log->content, '\n', &header_line)) {
+ // At least with glibc, sscanf will touch every byte of the string it scans,
+ // so make a copy of the first line for sscanf to use. (Maybe sscanf is
+ // calling strlen internally?)
+ sscanf((header_line.AsString()).c_str(), kFileSignature,
+ &log->log_version);
+ }
- if (log_version < kOldestSupportedVersion) {
- *err = ("build log version invalid, perhaps due to being too old; "
- "starting over");
- fclose(file);
- unlink(path.c_str());
- // Don't report this as a failure. An empty build log will cause
- // us to rebuild the outputs anyway.
- return true;
+ if (log->log_version < kOldestSupportedVersion) {
+ *err = ("build log version invalid, perhaps due to being too old; "
+ "starting over");
+ log->content = {};
+ log->file.reset();
+ unlink(path.c_str());
+ // Don't report this as a failure. An empty build log will cause
+ // us to rebuild the outputs anyway.
+ }
+
+ return true;
+}
+
+/// Split the build log's content (i.e. the lines excluding the header) into
+/// chunks. Each chunk is guaranteed to end with a newline character. Any output
+/// beyond the end of the last newline is quietly discarded.
+static std::vector<StringPiece> SplitBuildLog(StringPiece content) {
+ // The log file should end with an LF character, but if it doesn't, start by
+ // stripping off non-LF characters.
+ while (!content.empty() && content.back() != '\n') {
+ content.remove_suffix(1);
+ }
+
+ size_t ideal_chunk_count = GetOptimalThreadPoolJobCount();
+ size_t ideal_chunk_size = content.size() / ideal_chunk_count + 1;
+
+ std::vector<StringPiece> result;
+ StringPiece chunk;
+ while (GetNextPiece(&content, '\n', &chunk, ideal_chunk_size)) {
+ result.push_back(chunk);
+ }
+ if (!content.empty()) {
+ result.push_back(content);
+ }
+
+ return result;
+}
+
+/// Call the given function on each line in the given chunk. The chunk must end
+/// with an LF character. The LF characters are included in the string views
+/// passed to the callback.
+template <typename Func>
+static void VisitEachLineInChunk(StringPiece chunk, Func func) {
+ assert(!chunk.empty() && chunk.back() == '\n');
+ StringPiece line;
+ while (GetNextPiece(&chunk, '\n', &line)) {
+ func(line);
+ }
+}
+
+/// Count the number of LF newline characters in the string. The string is
+/// guaranteed to end with an LF.
+static size_t CountNewlinesInChunk(StringPiece chunk) {
+ size_t line_count = 0;
+ VisitEachLineInChunk(chunk, [&line_count](StringPiece line) {
+ ++line_count;
+ });
+ return line_count;
+}
+
+struct ParsedLine {
+ /// These fields are guaranteed to be followed by a whitespace character
+ /// (either a tab or an LF), but the whitespace terminator isn't explicitly
+ /// part of the string piece.
+ StringPiece start_time;
+ StringPiece end_time;
+ StringPiece mtime;
+ StringPiece path;
+ StringPiece command_hash;
+};
+
+/// Given a single line of the build log (including the terminator LF), split
+/// the line into its various tab-delimited fields.
+static inline bool SplitLine(StringPiece line, ParsedLine* out) {
+ assert(!line.empty() && line.back() == '\n');
+ line.remove_suffix(1);
+
+ auto get_next_field = [&line](StringPiece* out_piece) {
+ // Extract the next piece from the line. If we're successful, remove the
+ // field separator from the end of the piece.
+ if (!GetNextPiece(&line, kFieldSeparator, out_piece)) return false;
+ assert(!out_piece->empty() && out_piece->back() == kFieldSeparator);
+ out_piece->remove_suffix(1);
+ return true;
+ };
+
+ *out = {};
+ if (!get_next_field(&out->start_time)) return false;
+ if (!get_next_field(&out->end_time)) return false;
+ if (!get_next_field(&out->mtime)) return false;
+ if (!get_next_field(&out->path)) return false;
+ out->command_hash = line;
+
+ return true;
+}
+
+/// Given a single line of the build log (including the terminator LF), return
+/// just the path field.
+static StringPiece GetPathForLine(StringPiece line) {
+ ParsedLine parsed_line;
+ return SplitLine(line, &parsed_line) ? parsed_line.path : StringPiece();
+}
+
+bool BuildLog::Load(const string& path, string* err) {
+ METRIC_RECORD(".ninja_log load");
+ assert(entries_.empty());
+
+ BuildLogInput log;
+ if (!OpenBuildLogForReading(path, &log, err)) return false;
+ if (log.file.get() == nullptr) return true;
+
+ std::vector<StringPiece> chunks = SplitBuildLog(log.content);
+ std::unique_ptr<ThreadPool> thread_pool = CreateThreadPool();
+
+ // The concurrent hashmap doesn't automatically resize when entries are added
+ // to it, so we need to reserve space in advance. The number of newlines
+ // should exceed the number of distinct paths in the build log by a small
+ // factor.
+ size_t line_count = 0;
+ for (size_t count :
+ ParallelMap(thread_pool.get(), chunks, CountNewlinesInChunk)) {
+ line_count += count;
+ }
+ entries_.reserve(line_count);
+
+ // Construct an initial table of path -> LogEntry. Each LogEntry is
+ // initialized with the newest record for a particular path. Build a list of
+ // each chunk's new log entries.
+ std::vector<std::vector<LogEntry*>> chunk_new_entries =
+ ParallelMap(thread_pool.get(), chunks, [this, &log](StringPiece chunk) {
+ std::vector<LogEntry*> chunk_entries_list;
+ VisitEachLineInChunk(chunk, [&](StringPiece line) {
+ HashedStrView path = GetPathForLine(line);
+ if (path.empty()) return;
+ LogEntry* entry = nullptr;
+ if (LogEntry** entry_it = entries_.Lookup(path)) {
+ entry = *entry_it;
+ } else {
+ std::unique_ptr<LogEntry> new_entry(new LogEntry(path));
+ new_entry->newest_parsed_line = line.data() - log.content.data();
+ if (entries_.insert({ new_entry->output, new_entry.get() }).second) {
+ chunk_entries_list.push_back(new_entry.release());
+ return;
+ } else {
+ // Another thread beat us to it. Update the existing entry instead.
+ LogEntry** entry_it = entries_.Lookup(path);
+ assert(entry_it != nullptr);
+ entry = *entry_it;
+ }
}
- }
+ AtomicUpdateMaximum<size_t>(&entry->newest_parsed_line,
+ line.data() - log.content.data());
+ });
+ return chunk_entries_list;
+ });
- // If no newline was found in this chunk, read the next.
- if (!line_end)
- continue;
-
- const char kFieldSeparator = '\t';
-
- char* start = line_start;
- char* end = (char*)memchr(start, kFieldSeparator, line_end - start);
- if (!end)
- continue;
- *end = 0;
-
- int start_time = 0, end_time = 0;
- TimeStamp restat_mtime = 0;
-
- start_time = atoi(start);
- start = end + 1;
-
- end = (char*)memchr(start, kFieldSeparator, line_end - start);
- if (!end)
- continue;
- *end = 0;
- end_time = atoi(start);
- start = end + 1;
-
- end = (char*)memchr(start, kFieldSeparator, line_end - start);
- if (!end)
- continue;
- *end = 0;
- restat_mtime = strtoll(start, NULL, 10);
- start = end + 1;
-
- end = (char*)memchr(start, kFieldSeparator, line_end - start);
- if (!end)
- continue;
- string output = string(start, end - start);
-
- start = end + 1;
- end = line_end;
-
- LogEntry* entry;
- Entries::iterator i = entries_.find(output);
- if (i != entries_.end()) {
- entry = i->second;
- } else {
- entry = new LogEntry(output);
- entries_.insert(Entries::value_type(entry->output, entry));
- ++unique_entry_count;
- }
- ++total_entry_count;
-
- entry->start_time = start_time;
- entry->end_time = end_time;
- entry->mtime = restat_mtime;
- if (log_version >= 5) {
- char c = *end; *end = '\0';
- entry->command_hash = (uint64_t)strtoull(start, NULL, 16);
- *end = c;
- } else {
- entry->command_hash = LogEntry::HashCommand(StringPiece(start,
- end - start));
- }
+ // Collect all the log entries into a flat vector so we can distribute the
+ // final parsing work. This list has a non-deterministic order when a node
+ // appears in multiple chunks.
+ std::vector<LogEntry*> entries_vec;
+ entries_vec.reserve(entries_.size());
+ for (auto& chunk : chunk_new_entries) {
+ std::move(chunk.begin(), chunk.end(), std::back_inserter(entries_vec));
}
- fclose(file);
- if (!line_start) {
- return true; // file was empty
- }
+ // Finish parsing the log entries.
+ ParallelMap(thread_pool.get(), entries_vec, [&log](LogEntry* entry) {
+ // Locate the end of the line. The end of the line wasn't stored in the log
+ // entry because that would complicate the atomic line location updating.
+ assert(entry->newest_parsed_line < log.content.size());
+ StringPiece line_to_end = log.content.substr(entry->newest_parsed_line);
+ StringPiece line;
+ if (!GetNextPiece(&line_to_end, '\n', &line)) {
+ assert(false && "build log changed during parsing");
+ abort();
+ }
+
+ ParsedLine parsed_line {};
+ if (!SplitLine(line, &parsed_line)) {
+ assert(false && "build log changed during parsing");
+ abort();
+ }
+
+ // Initialize the entry object.
+ entry->start_time = atoi(parsed_line.start_time.data());
+ entry->end_time = atoi(parsed_line.end_time.data());
+ entry->mtime = strtoll(parsed_line.mtime.data(), nullptr, 10);
+ entry->command_hash = static_cast<uint64_t>(
+ strtoull(parsed_line.command_hash.data(), nullptr, 16));
+ });
+
+ int total_entry_count = line_count;
+ int unique_entry_count = entries_vec.size();
// Decide whether it's time to rebuild the log:
// - if we're upgrading versions
// - if it's getting large
- int kMinCompactionEntryCount = 100;
- int kCompactionRatio = 3;
- if (log_version < kCurrentVersion) {
+ const int kMinCompactionEntryCount = 100;
+ const int kCompactionRatio = 3;
+ if (log.log_version < kCurrentVersion) {
needs_recompaction_ = true;
} else if (total_entry_count > kMinCompactionEntryCount &&
total_entry_count > unique_entry_count * kCompactionRatio) {
@@ -357,11 +475,10 @@
return true;
}
-BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) {
- Entries::iterator i = entries_.find(path);
- if (i != entries_.end())
- return i->second;
- return NULL;
+BuildLog::LogEntry* BuildLog::LookupByOutput(const HashedStrView& path) {
+ if (LogEntry** i = entries_.Lookup(path))
+ return *i;
+ return nullptr;
}
bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) {
@@ -388,22 +505,23 @@
return false;
}
- vector<StringPiece> dead_outputs;
- for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
- if (user.IsPathDead(i->first)) {
- dead_outputs.push_back(i->first);
- continue;
- }
+ Entries new_entries;
+ new_entries.reserve(std::max(entries_.size(), entries_.bucket_count()));
- if (!WriteEntry(f, *i->second)) {
+ for (const std::pair<HashedStrView, LogEntry*>& pair : entries_) {
+ if (user.IsPathDead(pair.first.str_view()))
+ continue;
+
+ new_entries.insert(pair);
+
+ if (!WriteEntry(f, *pair.second)) {
*err = strerror(errno);
fclose(f);
return false;
}
}
- for (size_t i = 0; i < dead_outputs.size(); ++i)
- entries_.erase(dead_outputs[i]);
+ entries_.swap(new_entries);
fclose(f);
if (unlink(path.c_str()) < 0) {
diff --git a/src/build_log.h b/src/build_log.h
index 5268fab..79bbd74 100644
--- a/src/build_log.h
+++ b/src/build_log.h
@@ -19,7 +19,8 @@
#include <stdio.h>
using namespace std;
-#include "hash_map.h"
+#include "hashed_str_view.h"
+#include "concurrent_hash_map.h"
#include "timestamp.h"
#include "util.h" // uint64_t
@@ -52,12 +53,15 @@
bool Load(const string& path, string* err);
struct LogEntry {
- string output;
+ HashedStr output;
uint64_t command_hash;
int start_time;
int end_time;
TimeStamp mtime;
+ // Used during build log parsing.
+ std::atomic<size_t> newest_parsed_line;
+
static uint64_t HashCommand(StringPiece command);
// Used by tests.
@@ -67,13 +71,13 @@
mtime == o.mtime;
}
- explicit LogEntry(const string& output);
- LogEntry(const string& output, uint64_t command_hash,
+ explicit LogEntry(const HashedStrView& output);
+ LogEntry(const HashedStrView& output, uint64_t command_hash,
int start_time, int end_time, TimeStamp restat_mtime);
};
/// Lookup a previously-run command by its output path.
- LogEntry* LookupByOutput(const string& path);
+ LogEntry* LookupByOutput(const HashedStrView& path);
/// Serialize an entry into a log file.
bool WriteEntry(FILE* f, const LogEntry& entry);
@@ -81,7 +85,10 @@
/// Rewrite the known log entries, throwing away old data.
bool Recompact(const string& path, const BuildLogUser& user, string* err);
- typedef ExternalStringHashMap<LogEntry*>::Type Entries;
+ /// The HashedStrView refers to memory in the LogEntry itself. This map uses
+ /// a value of LogEntry* rather than LogEntry, because moving a LogEntry would
+ /// invalidate the HashedStrView.
+ typedef ConcurrentHashMap<HashedStrView, LogEntry*> Entries;
const Entries& entries() const { return entries_; }
private:
diff --git a/src/build_log_test.cc b/src/build_log_test.cc
index ad30380..f5e8c56 100644
--- a/src/build_log_test.cc
+++ b/src/build_log_test.cc
@@ -66,7 +66,7 @@
ASSERT_TRUE(e2);
ASSERT_TRUE(*e1 == *e2);
ASSERT_EQ(15, e1->start_time);
- ASSERT_EQ("out", e1->output);
+ ASSERT_EQ("out", e1->output.str());
}
TEST_F(BuildLogTest, FirstWriteAddsSignature) {
@@ -101,9 +101,9 @@
TEST_F(BuildLogTest, DoubleEntry) {
FILE* f = fopen(kTestFilename, "wb");
- fprintf(f, "# ninja log v4\n");
- fprintf(f, "0\t1\t2\tout\tcommand abc\n");
- fprintf(f, "3\t4\t5\tout\tcommand def\n");
+ fprintf(f, "# ninja log v5\n");
+ fprintf(f, "0\t1\t2\tout\t67ab\n");
+ fprintf(f, "3\t4\t5\tout\t89cd\n");
fclose(f);
string err;
@@ -113,7 +113,7 @@
BuildLog::LogEntry* e = log.LookupByOutput("out");
ASSERT_TRUE(e);
- ASSERT_NO_FATAL_FAILURE(AssertHash("command def", e->command_hash));
+ ASSERT_EQ(0x89cd, e->command_hash);
}
TEST_F(BuildLogTest, Truncate) {
@@ -166,34 +166,19 @@
ASSERT_NE(err.find("version"), string::npos);
}
-TEST_F(BuildLogTest, SpacesInOutputV4) {
- FILE* f = fopen(kTestFilename, "wb");
- fprintf(f, "# ninja log v4\n");
- fprintf(f, "123\t456\t456\tout with space\tcommand\n");
- fclose(f);
-
- string err;
- BuildLog log;
- EXPECT_TRUE(log.Load(kTestFilename, &err));
- ASSERT_EQ("", err);
-
- BuildLog::LogEntry* e = log.LookupByOutput("out with space");
- ASSERT_TRUE(e);
- ASSERT_EQ(123, e->start_time);
- ASSERT_EQ(456, e->end_time);
- ASSERT_EQ(456, e->mtime);
- ASSERT_NO_FATAL_FAILURE(AssertHash("command", e->command_hash));
-}
-
TEST_F(BuildLogTest, DuplicateVersionHeader) {
// Old versions of ninja accidentally wrote multiple version headers to the
// build log on Windows. This shouldn't crash, and the second version header
// should be ignored.
+ //
+ // This test initially tested a v4 log, but support for v4 was removed. It now
+ // tests a v5 log, but that doesn't mean that the original duplicate-header
+ // bug ever affected v5 logs.
FILE* f = fopen(kTestFilename, "wb");
- fprintf(f, "# ninja log v4\n");
- fprintf(f, "123\t456\t456\tout\tcommand\n");
- fprintf(f, "# ninja log v4\n");
- fprintf(f, "456\t789\t789\tout2\tcommand2\n");
+ fprintf(f, "# ninja log v5\n");
+ fprintf(f, "123\t456\t456\tout\t12ab\n");
+ fprintf(f, "# ninja log v5\n");
+ fprintf(f, "456\t789\t789\tout2\t34cd\n");
fclose(f);
string err;
@@ -206,26 +191,27 @@
ASSERT_EQ(123, e->start_time);
ASSERT_EQ(456, e->end_time);
ASSERT_EQ(456, e->mtime);
- ASSERT_NO_FATAL_FAILURE(AssertHash("command", e->command_hash));
+ ASSERT_EQ(0x12ab, e->command_hash);
e = log.LookupByOutput("out2");
ASSERT_TRUE(e);
ASSERT_EQ(456, e->start_time);
ASSERT_EQ(789, e->end_time);
ASSERT_EQ(789, e->mtime);
- ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash));
+ ASSERT_EQ(0x34cd, e->command_hash);
}
TEST_F(BuildLogTest, VeryLongInputLine) {
- // Ninja's build log buffer is currently 256kB. Lines longer than that are
- // silently ignored, but don't affect parsing of other lines.
+ // The parallelized build log parser accepts paths of arbitrary length.
+ std::string very_long_path;
+ for (size_t i = 0; i < (512 << 10) / strlen("/more_path"); ++i)
+ very_long_path += "/more_path";
+
FILE* f = fopen(kTestFilename, "wb");
- fprintf(f, "# ninja log v4\n");
- fprintf(f, "123\t456\t456\tout\tcommand start");
- for (size_t i = 0; i < (512 << 10) / strlen(" more_command"); ++i)
- fputs(" more_command", f);
+ fprintf(f, "# ninja log v5\n");
+ fprintf(f, "123\t456\t456\t%s\t1234abcd\n", very_long_path.c_str());
fprintf(f, "\n");
- fprintf(f, "456\t789\t789\tout2\tcommand2\n");
+ fprintf(f, "456\t789\t789\tout2\t2345bcde\n");
fclose(f);
string err;
@@ -233,15 +219,19 @@
EXPECT_TRUE(log.Load(kTestFilename, &err));
ASSERT_EQ("", err);
- BuildLog::LogEntry* e = log.LookupByOutput("out");
- ASSERT_EQ(NULL, e);
+ BuildLog::LogEntry* e = log.LookupByOutput(very_long_path.c_str());
+ ASSERT_TRUE(e);
+ ASSERT_EQ(123, e->start_time);
+ ASSERT_EQ(456, e->end_time);
+ ASSERT_EQ(456, e->mtime);
+ ASSERT_EQ(0x1234abcd, e->command_hash);
e = log.LookupByOutput("out2");
ASSERT_TRUE(e);
ASSERT_EQ(456, e->start_time);
ASSERT_EQ(789, e->end_time);
ASSERT_EQ(789, e->mtime);
- ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash));
+ ASSERT_EQ(0x2345bcde, e->command_hash);
}
TEST_F(BuildLogTest, MultiTargetEdge) {
@@ -256,8 +246,8 @@
ASSERT_TRUE(e1);
BuildLog::LogEntry* e2 = log.LookupByOutput("out.d");
ASSERT_TRUE(e2);
- ASSERT_EQ("out", e1->output);
- ASSERT_EQ("out.d", e2->output);
+ ASSERT_EQ("out", e1->output.str());
+ ASSERT_EQ("out.d", e2->output.str());
ASSERT_EQ(21, e1->start_time);
ASSERT_EQ(21, e2->start_time);
ASSERT_EQ(22, e2->end_time);
@@ -297,11 +287,11 @@
// "out2" is dead, it should've been removed.
BuildLog log3;
- EXPECT_TRUE(log2.Load(kTestFilename, &err));
+ EXPECT_TRUE(log3.Load(kTestFilename, &err));
ASSERT_EQ("", err);
- ASSERT_EQ(1u, log2.entries().size());
- ASSERT_TRUE(log2.LookupByOutput("out"));
- ASSERT_FALSE(log2.LookupByOutput("out2"));
+ ASSERT_EQ(1u, log3.entries().size());
+ ASSERT_TRUE(log3.LookupByOutput("out"));
+ ASSERT_FALSE(log3.LookupByOutput("out2"));
}
} // anonymous namespace
diff --git a/src/build_test.cc b/src/build_test.cc
index b5dbc6c..6b8e178 100644
--- a/src/build_test.cc
+++ b/src/build_test.cc
@@ -19,6 +19,7 @@
#include "build_log.h"
#include "deps_log.h"
#include "graph.h"
+#include "status.h"
#include "test.h"
struct CompareEdgesByOutput {
@@ -482,10 +483,56 @@
VirtualFileSystem* fs_;
};
+/// Fake implementation of Status, useful for tests.
+struct FakeStatus : public Status {
+ explicit FakeStatus() : last_output_(), warnings_() {}
+
+ virtual void PlanHasTotalEdges(int total) {}
+ virtual void BuildEdgeStarted(Edge* edge, int64_t start_time_millis) {}
+ virtual void BuildLoadDyndeps() {}
+ virtual void BuildStarted() {}
+ virtual void BuildFinished() {}
+
+ virtual void Debug(const char* msg, ...) {}
+ virtual void Info(const char* msg, ...) {}
+ virtual void Error(const char* msg, ...) {}
+
+ virtual void BuildEdgeFinished(Edge* edge, int64_t end_time_millis,
+ const CommandRunner::Result* result) {
+ last_output_ = result->output;
+ }
+
+ virtual void Warning(const char* msg, ...) {
+ va_list ap, ap2;
+ va_start(ap, msg);
+ va_copy(ap2, ap);
+
+ int len = vsnprintf(NULL, 0, msg, ap2);
+ va_end(ap2);
+ if (len < 0) {
+ warnings_.push_back("vsnprintf failed");
+ return;
+ }
+
+ string buf;
+ buf.resize(len + 1);
+
+ len = vsnprintf(&buf[0], len + 1, msg, ap);
+ buf.resize(len);
+
+ warnings_.push_back(buf);
+
+ va_end(ap);
+ }
+
+ string last_output_;
+ vector<string> warnings_;
+};
+
struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser {
BuildTest() : config_(MakeConfig()), command_runner_(&fs_),
- builder_(&state_, config_, NULL, NULL, &fs_),
- status_(config_) {
+ builder_(&state_, config_, NULL, NULL, &fs_, &status_, 0),
+ status_() {
}
virtual void SetUp() {
@@ -528,7 +575,7 @@
VirtualFileSystem fs_;
Builder builder_;
- BuildStatus status_;
+ FakeStatus status_;
};
void BuildTest::RebuildTarget(const string& target, const char* manifest,
@@ -552,12 +599,12 @@
DepsLog deps_log, *pdeps_log = NULL;
if (deps_path) {
ASSERT_TRUE(deps_log.Load(deps_path, pstate, &err));
- ASSERT_TRUE(deps_log.OpenForWrite(deps_path, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite(deps_path, fs_, &err));
ASSERT_EQ("", err);
pdeps_log = &deps_log;
}
- Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_);
+ Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_, &status_, 0);
EXPECT_TRUE(builder.AddTarget(target, &err));
command_runner_.commands_ran_.clear();
@@ -592,7 +639,9 @@
} else if (edge->rule().name() == "true" ||
edge->rule().name() == "fail" ||
edge->rule().name() == "interrupt" ||
- edge->rule().name() == "console") {
+ edge->rule().name() == "console" ||
+ edge->rule().name() == "mkdir" ||
+ edge->rule().name() == "phony_out") {
// Don't do anything.
} else if (edge->rule().name() == "cp") {
assert(!edge->inputs_.empty());
@@ -643,6 +692,18 @@
return true;
}
+ if (edge->rule().name() == "mkdir") {
+ result->status = ExitSuccess;
+ for (vector<Node*>::iterator out = edge->outputs_.begin();
+ out != edge->outputs_.end(); ++out) {
+ if (!fs_->MakeDir((*out)->path())) {
+ result->status = ExitFailure;
+ }
+ }
+ active_edges_.erase(edge_iter);
+ return true;
+ }
+
if (edge->rule().name() == "fail" ||
(edge->rule().name() == "touch-fail-tick2" && fs_->now_ == 2))
result->status = ExitFailure;
@@ -915,6 +976,23 @@
ASSERT_EQ("cc foo.c", edge->EvaluateCommand());
}
+TEST_F(BuildTest, DepFileOKWithPhonyOutputs) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n command = cc $in\n depfile = $out.d\n"
+"build foo.o: cc foo.c\n"));
+
+ config_.uses_phony_outputs = true;
+
+ fs_.Create("foo.c", "");
+ GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing.
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
+ EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ ASSERT_EQ("", err);
+
+ ASSERT_TRUE(GetNode("bar.h")->in_edge()->phony_from_depfile_);
+}
+
TEST_F(BuildTest, DepFileParseError) {
string err;
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
@@ -935,7 +1013,7 @@
"build b: touch || c\n"
"build a: touch | b || c\n"));
- vector<Edge*> c_out = GetNode("c")->out_edges();
+ vector<Edge*> c_out = GetNode("c")->GetOutEdges();
ASSERT_EQ(2u, c_out.size());
EXPECT_EQ("b", c_out[0]->outputs_[0]->path());
EXPECT_EQ("a", c_out[1]->outputs_[0]->path());
@@ -1384,8 +1462,8 @@
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- EXPECT_EQ("[3/3]", builder_.status_->FormatProgressStatus("[%s/%t]",
- BuildStatus::kEdgeStarted));
+ EXPECT_EQ(3u, command_runner_.commands_ran_.size());
+ EXPECT_EQ(3u, builder_.plan_.command_edge_count());
command_runner_.commands_ran_.clear();
state_.Reset();
@@ -1823,20 +1901,6 @@
ASSERT_EQ(1u, command_runner_.commands_ran_.size());
}
-TEST_F(BuildTest, StatusFormatElapsed) {
- status_.BuildStarted();
- // Before any task is done, the elapsed time must be zero.
- EXPECT_EQ("[%/e0.000]",
- status_.FormatProgressStatus("[%%/e%e]",
- BuildStatus::kEdgeStarted));
-}
-
-TEST_F(BuildTest, StatusFormatReplacePlaceholder) {
- EXPECT_EQ("[%/s0/t0/r0/u0/f0]",
- status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]",
- BuildStatus::kEdgeStarted));
-}
-
TEST_F(BuildTest, FailedDepsParse) {
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build bad_deps.o: cat in1\n"
@@ -1893,10 +1957,10 @@
// Run the build once, everything should be ok.
DepsLog deps_log;
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
EXPECT_TRUE(builder.AddTarget("out", &err));
ASSERT_EQ("", err);
@@ -1924,9 +1988,9 @@
// Run the build again.
DepsLog deps_log;
ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
command_runner_.commands_ran_.clear();
EXPECT_TRUE(builder.AddTarget("out", &err));
@@ -1964,10 +2028,10 @@
// Run the build once, everything should be ok.
DepsLog deps_log;
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
EXPECT_TRUE(builder.AddTarget("out", &err));
ASSERT_EQ("", err);
@@ -1994,9 +2058,9 @@
DepsLog deps_log;
ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
command_runner_.commands_ran_.clear();
EXPECT_TRUE(builder.AddTarget("out", &err));
@@ -2032,7 +2096,7 @@
// The deps log is NULL in dry runs.
config_.dry_run = true;
- Builder builder(&state, config_, NULL, NULL, &fs_);
+ Builder builder(&state, config_, NULL, NULL, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
command_runner_.commands_ran_.clear();
@@ -2087,10 +2151,10 @@
// Run the build once, everything should be ok.
DepsLog deps_log;
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
EXPECT_TRUE(builder.AddTarget("out", &err));
ASSERT_EQ("", err);
@@ -2114,9 +2178,9 @@
// Run the build again.
DepsLog deps_log;
ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
command_runner_.commands_ran_.clear();
EXPECT_TRUE(builder.AddTarget("out", &err));
@@ -2146,10 +2210,10 @@
// Run the build once, everything should be ok.
DepsLog deps_log;
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
EXPECT_TRUE(builder.AddTarget("fo o.o", &err));
ASSERT_EQ("", err);
@@ -2167,10 +2231,10 @@
DepsLog deps_log;
ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
Edge* edge = state.edges_.back();
@@ -2208,10 +2272,10 @@
// Run the build once, everything should be ok.
DepsLog deps_log;
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err));
ASSERT_EQ("", err);
@@ -2231,10 +2295,10 @@
DepsLog deps_log;
ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
Edge* edge = state.edges_.back();
@@ -2676,6 +2740,40 @@
EXPECT_EQ("touch out", command_runner_.commands_ran_[2]);
}
+TEST_F(BuildTest, DyndepBuildDiscoverNewInputWithValidation) {
+ // Verify that a dyndep file can be built and loaded to discover
+ // a new input to an edge.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build in: touch |@ validation\n"
+"build validation: touch in out\n"
+"build out: touch || dd\n"
+" dyndep = dd\n"
+ ));
+ fs_.Create("dd-in",
+"ninja_dyndep_version = 1\n"
+"build out: dyndep | in\n"
+);
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(4u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch in", command_runner_.commands_ran_[1]);
+ EXPECT_EQ("touch out", command_runner_.commands_ran_[2]);
+ EXPECT_EQ("touch validation", command_runner_.commands_ran_[3]);
+}
+
TEST_F(BuildTest, DyndepBuildDiscoverImplicitConnection) {
// Verify that a dyndep file can be built and loaded to discover
// that one edge has an implicit output that is also an implicit
@@ -3077,3 +3175,518 @@
EXPECT_EQ("touch tmp", command_runner_.commands_ran_[3]);
EXPECT_EQ("touch out", command_runner_.commands_ran_[4]);
}
+
+TEST_F(BuildTest, OutputDirectoryIgnored) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule mkdir\n"
+" command = mkdir $out\n"
+"build outdir: mkdir\n"));
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outdir", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputDirectoryWarning) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule mkdir\n"
+" command = mkdir $out\n"
+"build outdir: mkdir\n"));
+
+ config_.uses_phony_outputs = true;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outdir", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("ninja: outputs should be files, not directories: outdir", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputDirectoryError) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule mkdir\n"
+" command = mkdir $out\n"
+"build outdir: mkdir\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.output_directory_should_err = true;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outdir", &err));
+ EXPECT_EQ("", err);
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+
+ EXPECT_EQ("ninja: outputs should be files, not directories: outdir", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputFileMissingIgnore) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build outfile: true\n"));
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outfile", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputFileMissingWarning) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build outfile: true\n"));
+
+ config_.uses_phony_outputs = true;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outfile", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("ninja: output file missing after successful execution: outfile", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputFileMissingError) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build outfile: true\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.missing_output_file_should_err = true;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outfile", &err));
+ EXPECT_EQ("", err);
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+
+ EXPECT_EQ("ninja: output file missing after successful execution: outfile", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputFileNotNeeded) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build outphony: phony_out\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.missing_output_file_should_err = true;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outphony", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+}
+
+TEST_F(BuildWithLogTest, OldOutputFileIgnored) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build out: true in\n"));
+
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("", status_.last_output_);
+}
+
+TEST_F(BuildWithLogTest, OldOutputFileWarning) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build out: true in\n"));
+
+ config_.uses_phony_outputs = true;
+
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ("ninja: Missing `restat`? An output file is older than the most recent input:\n output: out\n input: in", status_.last_output_);
+}
+
+TEST_F(BuildWithLogTest, OldOutputFileError) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build out: true in\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.old_output_should_err = true;
+
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+
+ EXPECT_EQ("ninja: Missing `restat`? An output file is older than the most recent input:\n output: out\n input: in", status_.last_output_);
+}
+
+TEST_F(BuildWithLogTest, OutputFileUpdated) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n command = touch ${out}\n"
+"build out: touch in\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.old_output_should_err = true;
+
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("", status_.last_output_);
+}
+
+TEST_F(BuildWithDepsLogTest, MissingDepfileWarning) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out1: cat in1\n"
+" deps = gcc\n"
+" depfile = in1.d\n"));
+
+ string err;
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state_, config_, NULL, &deps_log, &fs_, &status_, 0);
+ builder.command_runner_.reset(&command_runner_);
+
+ EXPECT_TRUE(builder.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder.Build(&err));
+
+ ASSERT_EQ(1u, status_.warnings_.size());
+ ASSERT_EQ("depfile is missing (in1.d for out1)", status_.warnings_[0]);
+
+ builder.command_runner_.release();
+}
+
+TEST_F(BuildWithDepsLogTest, MissingDepfileError) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out1: cat in1\n"
+" deps = gcc\n"
+" depfile = in1.d\n"));
+
+ config_.missing_depfile_should_err = true;
+
+ string err;
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state_, config_, NULL, &deps_log, &fs_, &status_, 0);
+ builder.command_runner_.reset(&command_runner_);
+
+ EXPECT_TRUE(builder.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(builder.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+
+ EXPECT_EQ("depfile is missing", status_.last_output_);
+
+ builder.command_runner_.release();
+}
+
+TEST_F(BuildTest, PreRemoveOutputs) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n command = touch ${out}\n"
+"build out: touch in\n"
+"build out2: touch out\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.pre_remove_output_files = true;
+
+ fs_.Create("out", "");
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ EXPECT_EQ("", err);
+
+ fs_.files_created_.clear();
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(2u, command_runner_.commands_ran_.size());
+ EXPECT_EQ(2u, fs_.files_created_.size());
+ EXPECT_EQ(1u, fs_.files_removed_.size());
+}
+
+TEST_F(BuildTest, PreRemoveOutputsWithPhonyOutputs) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build out: phony_out\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.pre_remove_output_files = true;
+
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ fs_.files_created_.clear();
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ(0u, fs_.files_created_.size());
+ EXPECT_EQ(0u, fs_.files_removed_.size());
+}
+
+TEST_F(BuildTest, Validation) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "build out: cat in |@ validate\n"
+ "build validate: cat in2\n"));
+
+ fs_.Create("in", "");
+ fs_.Create("in2", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // Test touching "in" only rebuilds "out" ("validate" doesn't depend on
+ // "out").
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ err.clear();
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+
+ // Test touching "in2" only rebuilds "validate" ("out" doesn't depend on
+ // "validate").
+ fs_.Tick();
+ fs_.Create("in2", "");
+
+ err.clear();
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, ValidationDependsOnOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "build out: cat in |@ validate\n"
+ "build validate: cat in2 | out\n"));
+
+ fs_.Create("in", "");
+ fs_.Create("in2", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // Test touching "in" rebuilds "out" and "validate".
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ err.clear();
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // Test touching "in2" only rebuilds "validate" ("out" doesn't depend on
+ // "validate").
+ fs_.Tick();
+ fs_.Create("in2", "");
+
+ err.clear();
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildWithDepsLogTest, ValidationThroughDepfile) {
+ const char* manifest =
+ "build out: cat in |@ validate\n"
+ "build validate: cat in2 | out\n"
+ "build out2: cat in3\n"
+ " deps = gcc\n"
+ " depfile = out2.d\n";
+
+ string err;
+
+ {
+ fs_.Create("in", "");
+ fs_.Create("in2", "");
+ fs_.Create("in3", "");
+ fs_.Create("out2.d", "out: out");
+
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
+ builder.command_runner_.reset(&command_runner_);
+
+ EXPECT_TRUE(builder.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // On the first build, only the out2 command is run.
+ EXPECT_EQ(command_runner_.commands_ran_.size(), 1);
+
+ // The deps file should have been removed.
+ EXPECT_EQ(0, fs_.Stat("out2.d", &err));
+
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+
+ fs_.Tick();
+ command_runner_.commands_ran_.clear();
+
+ {
+ fs_.Create("in2", "");
+ fs_.Create("in3", "");
+
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
+ builder.command_runner_.reset(&command_runner_);
+
+ EXPECT_TRUE(builder.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // The out and validate actions should have been run as well as out2.
+ ASSERT_EQ(command_runner_.commands_ran_.size(), 3);
+ EXPECT_EQ(command_runner_.commands_ran_[0], "cat in > out");
+ EXPECT_EQ(command_runner_.commands_ran_[1], "cat in2 > validate");
+ EXPECT_EQ(command_runner_.commands_ran_[2], "cat in3 > out2");
+
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+}
diff --git a/src/clean.cc b/src/clean.cc
index d1f221d..30f0a3e 100644
--- a/src/clean.cc
+++ b/src/clean.cc
@@ -108,10 +108,10 @@
for (vector<Edge*>::iterator e = state_->edges_.begin();
e != state_->edges_.end(); ++e) {
// Do not try to remove phony targets
- if ((*e)->is_phony())
+ if ((*e)->is_phony() || (*e)->IsPhonyOutput())
continue;
// Do not remove generator's files unless generator specified.
- if (!generator && (*e)->GetBindingBool("generator"))
+ if (!generator && (*e)->IsGenerator())
continue;
for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
out_node != (*e)->outputs_.end(); ++out_node) {
@@ -127,7 +127,7 @@
void Cleaner::DoCleanTarget(Node* target) {
if (Edge* e = target->in_edge()) {
// Do not try to remove phony targets
- if (!e->is_phony()) {
+ if (!e->is_phony() && !e->IsPhonyOutput()) {
Remove(target->path());
RemoveEdgeFiles(e);
}
@@ -203,6 +203,9 @@
for (vector<Edge*>::iterator e = state_->edges_.begin();
e != state_->edges_.end(); ++e) {
if ((*e)->rule().name() == rule->name()) {
+ if ((*e)->IsPhonyOutput()) {
+ continue;
+ }
for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
out_node != (*e)->outputs_.end(); ++out_node) {
Remove((*out_node)->path());
@@ -227,7 +230,8 @@
assert(rule);
Reset();
- const Rule* r = state_->bindings_.LookupRule(rule);
+ ScopePosition end_of_root { &state_->root_scope_, kLastDeclIndex };
+ const Rule* r = Scope::LookupRuleAtPos(rule, end_of_root);
if (r) {
CleanRule(r);
} else {
@@ -245,7 +249,8 @@
LoadDyndeps();
for (int i = 0; i < rule_count; ++i) {
const char* rule_name = rules[i];
- const Rule* rule = state_->bindings_.LookupRule(rule_name);
+ ScopePosition end_of_root { &state_->root_scope_, kLastDeclIndex };
+ const Rule* rule = Scope::LookupRuleAtPos(rule_name, end_of_root);
if (rule) {
if (IsVerbose())
printf("Rule %s\n", rule_name);
diff --git a/src/clean_test.cc b/src/clean_test.cc
index 45187f4..1785656 100644
--- a/src/clean_test.cc
+++ b/src/clean_test.cc
@@ -426,6 +426,35 @@
EXPECT_LT(0, fs_.Stat("phony", &err));
}
+TEST_F(CleanTest, CleanPhonyOutput) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build phout: phony_out t1 t2\n"
+"build t1: cat\n"
+"build t2: cat\n"));
+
+ fs_.Create("phout", "");
+ fs_.Create("t1", "");
+ fs_.Create("t2", "");
+
+ // Check that CleanAll does not remove "phout".
+ Cleaner cleaner(&state_, config_, &fs_);
+ EXPECT_EQ(0, cleaner.CleanAll());
+ EXPECT_EQ(2, cleaner.cleaned_files_count());
+ EXPECT_LT(0, fs_.Stat("phout", &err));
+
+ fs_.Create("t1", "");
+ fs_.Create("t2", "");
+
+ // Check that CleanTarget does not remove "phony".
+ EXPECT_EQ(0, cleaner.CleanTarget("phout"));
+ EXPECT_EQ(2, cleaner.cleaned_files_count());
+ EXPECT_LT(0, fs_.Stat("phout", &err));
+}
+
TEST_F(CleanTest, CleanDepFileAndRspFileWithSpaces) {
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"rule cc_dep\n"
diff --git a/src/concurrent_hash_map.h b/src/concurrent_hash_map.h
new file mode 100644
index 0000000..6e5d660
--- /dev/null
+++ b/src/concurrent_hash_map.h
@@ -0,0 +1,274 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_CONCURRENT_HASH_MAP_
+#define NINJA_CONCURRENT_HASH_MAP_
+
+#include <stdint.h>
+
+#include <atomic>
+#include <memory>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#include "metrics.h"
+#include "util.h"
+
+/// A hash table that allows for concurrent lookups and insertions.
+///
+/// Resizing the table isn't thread-safe; inserting items doesn't automatically
+/// resize the table.
+template <typename K, typename V>
+class ConcurrentHashMap {
+public:
+ using key_type = K;
+ using mapped_type = V;
+ using value_type = std::pair<const K, V>;
+ using reference = value_type&;
+ using pointer = value_type*;
+ using const_reference = const value_type&;
+ using const_pointer = const value_type*;
+
+private:
+ struct Node;
+
+ struct NodeStub {
+ std::atomic<Node*> next { nullptr };
+ };
+
+ struct Node : NodeStub {
+ Node(size_t hash, const value_type& value) : hash(hash), value(value) {}
+
+ size_t hash = 0;
+ value_type value {};
+ };
+
+ /// Record the number of elements in the hash map. Assign each thread to a
+ /// separate index within the size table to avoid false sharing among CPUs.
+ struct NINJA_ALIGNAS_CACHE_LINE SizeRecord {
+ std::atomic<ssize_t> size;
+ };
+ std::vector<SizeRecord> size_ { GetThreadSlotCount() };
+
+ void IncrementSize() {
+ thread_local size_t idx = GetThreadSlotIndex();
+ size_[idx].size++;
+ }
+
+public:
+ explicit ConcurrentHashMap(size_t size=1)
+ : table_(std::max<size_t>(size, 1)) {}
+
+ ConcurrentHashMap(const ConcurrentHashMap&) = delete;
+ ConcurrentHashMap& operator=(const ConcurrentHashMap&) = delete;
+
+ ~ConcurrentHashMap() {
+ ForEachNode(table_, [](Node* node) {
+ delete node;
+ });
+ }
+
+ void swap(ConcurrentHashMap& other) {
+ size_.swap(other.size_);
+ table_.swap(other.table_);
+ }
+
+ /// Atomically lookup a key.
+ V* Lookup(const K& key) {
+ std::pair<NodeStub*, Node*> pos = Find(std::hash<K>()(key), key);
+ return pos.first ? nullptr : &pos.second->value.second;
+ }
+
+ /// Atomically lookup a key.
+ const V* Lookup(const K& key) const {
+ auto self = const_cast<ConcurrentHashMap*>(this);
+ std::pair<NodeStub*, Node*> pos = self->Find(std::hash<K>()(key), key);
+ return pos.first ? nullptr : &pos.second->value.second;
+ }
+
+ /// Atomically insert a (K, V) entry into the hash map if K isn't already in
+ /// the map. Does nothing if K is already in the map.
+ ///
+ /// Returns the address of the V entry in the map, and a bool denoting whether
+ /// or not an insertion was performed.
+ ///
+ /// This function avoids returning an iterator because insertion is
+ /// thread-safe while iteration isn't, so returning an iterator seemed
+ /// hazardous.
+ std::pair<V*, bool> insert(const value_type& value) {
+ size_t hash = std::hash<K>()(value.first);
+ std::unique_ptr<Node> node(new Node(hash, value));
+ std::pair<Node*, bool> result = InsertNode(std::move(node));
+ return { &result.first->value.second, result.second };
+ }
+
+ size_t size() const {
+ ssize_t result = 0;
+ for (const SizeRecord& rec : size_)
+ result += rec.size.load();
+ return result;
+ }
+ bool empty() const { return size() == 0; }
+ size_t bucket_count() const { return table_.size(); }
+
+ /// Unlike insertion and lookup, rehashing is *not* thread-safe. This class
+ /// does not automatically rehash the table on insertion.
+ void rehash(size_t buckets) {
+ METRIC_RECORD("ConcurrentHashMap rehash");
+
+ std::vector<NodeStub> tmp_table(std::max<size_t>(buckets, 1));
+ tmp_table.swap(table_);
+ ForEachNode(tmp_table, [this](Node* node) {
+ node->next = nullptr;
+ InsertNode(std::unique_ptr<Node>(node));
+ });
+ }
+
+ void reserve(size_t count) {
+ if (count < bucket_count())
+ return;
+ rehash(count);
+ }
+
+private:
+ template <typename Func>
+ void ForEachNode(const std::vector<NodeStub>& table, Func&& func) {
+ for (const NodeStub& stub : table) {
+ Node* node = stub.next.load();
+ while (node != nullptr) {
+ Node* next = node->next.load();
+ func(node);
+ node = next;
+ }
+ }
+ }
+
+ /// Insert a node into the hash map if it isn't already there.
+ std::pair<Node*, bool> InsertNode(std::unique_ptr<Node> node) {
+ while (true) {
+ std::pair<NodeStub*, Node*> pos = Find(node->hash, node->value.first);
+ if (pos.first == nullptr)
+ return { pos.second, false };
+
+ node->next = pos.second;
+ Node* expected = pos.second;
+ if (pos.first->next.compare_exchange_weak(expected, node.get())) {
+ IncrementSize();
+ return { node.release(), true };
+ }
+ }
+ }
+
+ std::vector<NodeStub> table_;
+
+ /// Find either a matching node or the insertion point for the node. To ensure
+ /// deterministic behavior, the nodes are kept in ascending order, first by
+ /// the hash value, then by the key itself.
+ ///
+ /// This function returns a pair of (stub, node):
+ /// - If the node is found, stub is null and node is non-null.
+ /// - If the node isn't found, stub is non-null, and node is the loaded value
+ /// of the stub, which should be used in the CAS to insert the node.
+ std::pair<NodeStub*, Node*> Find(size_t hash, const K& key) {
+ NodeStub* stub = &table_[hash % table_.size()];
+ while (true) {
+ Node* ptr = stub->next.load();
+ if (ptr == nullptr || hash < ptr->hash) {
+ return { stub, ptr };
+ }
+ if (hash == ptr->hash) {
+ // In the common case, when the hash and ptr->hash are equal, we expect
+ // the keys to be equal too, so compare for equality before comparing
+ // for less-than.
+ if (key == ptr->value.first) {
+ return { nullptr, ptr };
+ } else if (key < ptr->value.first) {
+ return { stub, ptr };
+ }
+ }
+ stub = ptr;
+ }
+ }
+
+ struct MutCfg { using ValueType = value_type; };
+ struct ConstCfg { using ValueType = const value_type; };
+
+ /// Something like a ForwardIterator. Iteration over the table isn't
+ /// thread-safe.
+ template <typename Cfg>
+ class iterator_impl {
+ iterator_impl(const ConcurrentHashMap* container) : container_(container) {}
+ iterator_impl(const ConcurrentHashMap* container, size_t bucket, Node* node)
+ : container_(container),
+ bucket_(bucket),
+ node_(node) {}
+
+ const ConcurrentHashMap* container_ = nullptr;
+ size_t bucket_ = SIZE_MAX;
+ Node* node_ = nullptr;
+
+ public:
+ /// This constructor is a copy constructor for const_iterator and an
+ /// implicit conversion for const_iterator -> iterator.
+ iterator_impl(const iterator_impl<ConstCfg>& other)
+ : container_(other.container_),
+ bucket_(other.bucket_),
+ node_(other.node_) {}
+
+ typename Cfg::ValueType& operator*() { return node_->value; }
+ typename Cfg::ValueType* operator->() { return &node_->value; }
+
+ iterator_impl<Cfg>& operator++() {
+ node_ = node_->next.load();
+ if (node_ == nullptr)
+ *this = container_->FindNextOccupiedBucket(bucket_ + 1);
+ return *this;
+ }
+
+ bool operator==(const iterator_impl<Cfg>& other) const {
+ return node_ == other.node_;
+ }
+ bool operator!=(const iterator_impl<Cfg>& other) const {
+ return node_ != other.node_;
+ }
+
+ friend class ConcurrentHashMap;
+ };
+
+public:
+ using iterator = iterator_impl<MutCfg>;
+ using const_iterator = iterator_impl<ConstCfg>;
+
+ iterator begin() { return FindNextOccupiedBucket(0); }
+ iterator end() { return iterator { this }; }
+ const_iterator begin() const { return FindNextOccupiedBucket(0); }
+ const_iterator end() const { return const_iterator { this }; }
+ const_iterator cbegin() const { return FindNextOccupiedBucket(0); }
+ const_iterator cend() const { return const_iterator { this }; }
+
+private:
+ const_iterator FindNextOccupiedBucket(size_t idx) const {
+ while (idx < table_.size()) {
+ Node* node = table_[idx].next.load();
+ if (node != nullptr) {
+ return const_iterator { this, idx, node };
+ }
+ ++idx;
+ }
+ return end();
+ }
+};
+
+#endif // NINJA_CONCURRENT_HASH_MAP_
diff --git a/src/debug_flags.cc b/src/debug_flags.cc
index 44b14c4..e77d351 100644
--- a/src/debug_flags.cc
+++ b/src/debug_flags.cc
@@ -19,3 +19,5 @@
bool g_keep_rsp = false;
bool g_experimental_statcache = true;
+
+bool g_use_threads = true;
diff --git a/src/debug_flags.h b/src/debug_flags.h
index e08a43b..d2d57ee 100644
--- a/src/debug_flags.h
+++ b/src/debug_flags.h
@@ -30,4 +30,6 @@
extern bool g_experimental_statcache;
+extern bool g_use_threads;
+
#endif // NINJA_EXPLAIN_H_
diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc
index 405289f..16403f1 100644
--- a/src/depfile_parser.cc
+++ b/src/depfile_parser.cc
@@ -1,4 +1,4 @@
-/* Generated by re2c 1.1.1 */
+/* Generated by re2c 1.3 */
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/src/deps_log.cc b/src/deps_log.cc
index 4aaffeb..f2a3888 100644
--- a/src/deps_log.cc
+++ b/src/deps_log.cc
@@ -25,14 +25,21 @@
typedef unsigned __int32 uint32_t;
#endif
+#include <numeric>
+
+#include "disk_interface.h"
#include "graph.h"
#include "metrics.h"
+#include "parallel_map.h"
#include "state.h"
#include "util.h"
// The version is stored as 4 bytes after the signature and also serves as a
// byte order mark. Signature and version combined are 16 bytes long.
-const char kFileSignature[] = "# ninjadeps\n";
+static constexpr StringPiece kFileSignature { "# ninjadeps\n", 12 };
+static_assert(kFileSignature.size() % 4 == 0,
+ "file signature size is not a multiple of 4");
+static constexpr size_t kFileHeaderSize = kFileSignature.size() + 4;
const int kCurrentVersion = 4;
// Record size is currently limited to less than the full 32 bit, due to
@@ -43,9 +50,9 @@
Close();
}
-bool DepsLog::OpenForWrite(const string& path, string* err) {
+bool DepsLog::OpenForWrite(const string& path, const DiskInterface& disk, string* err) {
if (needs_recompaction_) {
- if (!Recompact(path, err))
+ if (!Recompact(path, disk, err))
return false;
}
@@ -64,7 +71,7 @@
fseek(file_, 0, SEEK_END);
if (ftell(file_) == 0) {
- if (fwrite(kFileSignature, sizeof(kFileSignature) - 1, 1, file_) < 1) {
+ if (fwrite(kFileSignature.data(), kFileSignature.size(), 1, file_) < 1) {
*err = strerror(errno);
return false;
}
@@ -167,123 +174,481 @@
file_ = NULL;
}
-bool DepsLog::Load(const string& path, State* state, string* err) {
- METRIC_RECORD(".ninja_deps load");
- char buf[kMaxRecordSize + 1];
- FILE* f = fopen(path.c_str(), "rb");
- if (!f) {
- if (errno == ENOENT)
- return true;
- *err = strerror(errno);
+struct DepsLogWordSpan {
+ size_t begin; // starting index into a deps log buffer of uint32 words
+ size_t end; // stopping index
+};
+
+/// Split the v4 deps log into independently-parseable chunks using a heuristic.
+/// If the heuristic fails, we'll still load the file correctly, but it could be
+/// slower.
+///
+/// There are two kinds of records -- path and deps records. Their formats:
+///
+/// path:
+/// - uint32 size -- high bit is clear
+/// - String content. The string is padded to a multiple of 4 bytes with
+/// trailing NULs.
+/// - uint32 checksum (ones complement of the path's index / node ID)
+///
+/// deps:
+/// - uint32 size -- high bit is set
+/// - int32 output_path_id
+/// - uint32 output_path_mtime_lo
+/// - uint32 output_path_mtime_hi
+/// - int32 input_path_id[...] -- every remaining word is an input ID
+///
+/// To split the deps log into chunks, look for uint32 words with the value
+/// 0x8000xxxx, where xxxx is nonzero. Such a word is almost guaranteed to be
+/// the size field of a deps record (with fewer than ~16K dependencies):
+/// - It can't be part of a string, because paths can't have embedded NULs.
+/// - It (probably) can't be a node ID, because node IDs are represented using
+/// "int", and it would be unlikely to have more than 2 billion of them. An
+/// Android build typically has about 1 million nodes.
+/// - It's unlikely to be part of a path checksum, because that would also
+/// imply that we have at least 2 billion nodes.
+/// - It could be the upper word of a mtime from 2262, or the lower word, which
+/// wraps every ~4s. We rule these out by looking for the mtime's deps size
+/// two or three words above the split candidate.
+///
+/// This heuristic can fail in a few ways:
+/// - We only find path records in the area we scan.
+/// - The deps records all have >16K of dependencies. (Almost all deps records
+/// I've seen in the Android build have a few hundred. Only a few have ~10K.)
+/// - All deps records with <16K of dependencies are preceded by something that
+/// the mtime check rejects:
+/// - A deps record with an mtime within a 524us span every 4s, with zero or
+/// one dependency.
+/// - A deps record with an mtime from 2262, with one or two dependencies.
+/// - A path record containing "xx xx 0[0-7] 80" (little-endian) or
+/// "80 0[0-7] xx xx" (big-endian) as the last or second-to-last word. The
+/// "0[0-7]" is a control character, and "0[0-7] 80" is invalid UTF-8.
+///
+/// Maybe we can add a delimiter to the log format and replace this heuristic.
+size_t MustBeDepsRecordHeader(DepsLogData log, size_t index) {
+ assert(index < log.size);
+
+ size_t this_header = log.words[index];
+ if ((this_header & 0xffff0000) != 0x80000000) return false;
+ if ((this_header & 0x0000ffff) == 0) return false;
+
+ // We've either found a deps record or an mtime. If it's an mtime, the
+ // word two or three spaces back will be a valid deps size (0x800xxxxx).
+ auto might_be_deps_record_header = [](size_t header) {
+ return (header & 0x80000000) == 0x80000000 &&
+ (header & 0x7fffffff) <= kMaxRecordSize;
+ };
+ if (index >= 3 && might_be_deps_record_header(log.words[index - 3])) {
+ return false;
+ }
+ if (index >= 2 && might_be_deps_record_header(log.words[index - 2])) {
return false;
}
- bool valid_header = true;
+ // Success: As long as the deps log is valid up to this point, this index
+ // must start a deps record.
+ return true;
+}
+
+static std::vector<DepsLogWordSpan>
+SplitDepsLog(DepsLogData log, ThreadPool* thread_pool) {
+ if (log.size == 0) return {};
+
+ std::vector<std::pair<size_t, size_t>> blind_splits =
+ SplitByThreads(log.size);
+ std::vector<DepsLogWordSpan> chunks;
+ size_t chunk_start = 0;
+
+ auto split_candidates = ParallelMap(thread_pool, blind_splits,
+ [log](std::pair<size_t, size_t> chunk) {
+ for (size_t i = chunk.first; i < chunk.second; ++i) {
+ if (MustBeDepsRecordHeader(log, i)) {
+ return i;
+ }
+ }
+ return SIZE_MAX;
+ });
+ for (size_t candidate : split_candidates) {
+ assert(chunk_start <= candidate);
+ if (candidate != SIZE_MAX && chunk_start < candidate) {
+ chunks.push_back({ chunk_start, candidate });
+ chunk_start = candidate;
+ }
+ }
+
+ assert(chunk_start < log.size);
+ chunks.push_back({ chunk_start, log.size });
+ return chunks;
+}
+
+struct InputRecord {
+ enum Kind { InvalidHeader, PathRecord, DepsRecord } kind = InvalidHeader;
+ size_t size = 0; // number of 32-bit words, including the header
+
+ union {
+ struct {
+ StringPiece path;
+ int checksum;
+ } path;
+
+ struct {
+ int output_id;
+ TimeStamp mtime;
+ const uint32_t* deps;
+ size_t deps_count;
+ } deps;
+ } u = {};
+};
+
+/// Parse a deps log record at the given index within the loaded deps log.
+/// If there is anything wrong with the header (e.g. the record extends past the
+/// end of the file), this function returns a blank InvalidHeader record.
+static InputRecord ParseRecord(DepsLogData log, size_t index) {
+ InputRecord result {};
+ assert(index < log.size);
+
+ // The header of a record is the size of the record, in bytes, without
+ // including the header itself. The high bit is set for a deps record and
+ // clear for a path record.
+ const uint32_t header = log.words[index];
+ const uint32_t raw_size = header & 0x7fffffff;
+ if (raw_size % sizeof(uint32_t) != 0) return result;
+ if (raw_size > kMaxRecordSize) return result;
+
+ // Add the header to the size for easier iteration over records later on.
+ const size_t size = raw_size / sizeof(uint32_t) + 1;
+ if (log.size - index < size) return result;
+
+ if ((header & 0x80000000) == 0) {
+ // Path record (header, content, checksum).
+ if (size < 3) return result;
+
+ const char* path = reinterpret_cast<const char*>(&log.words[index + 1]);
+ size_t path_size = (size - 2) * sizeof(uint32_t);
+ if (path[path_size - 1] == '\0') --path_size;
+ if (path[path_size - 1] == '\0') --path_size;
+ if (path[path_size - 1] == '\0') --path_size;
+
+ result.kind = InputRecord::PathRecord;
+ result.size = size;
+ result.u.path.path = StringPiece(path, path_size);
+ result.u.path.checksum = log.words[index + size - 1];
+ return result;
+ } else {
+ // Deps record (header, output_id, mtime_lo, mtime_hi).
+ if (size < 4) return result;
+
+ result.kind = InputRecord::DepsRecord;
+ result.size = size;
+ result.u.deps.output_id = log.words[index + 1];
+ result.u.deps.mtime =
+ (TimeStamp)(((uint64_t)(unsigned int)log.words[index + 3] << 32) |
+ (uint64_t)(unsigned int)log.words[index + 2]);
+ result.u.deps.deps = &log.words[index + 4];
+ result.u.deps.deps_count = size - 4;
+ return result;
+ }
+}
+
+struct DepsLogInputFile {
+ std::unique_ptr<LoadedFile> file;
+ DepsLogData data;
+};
+
+static bool OpenDepsLogForReading(const std::string& path,
+ DepsLogInputFile* log,
+ std::string* err) {
+ *log = {};
+
+ RealDiskInterface file_reader;
+ std::string load_err;
+ switch (file_reader.LoadFile(path, &log->file, &load_err)) {
+ case FileReader::Okay:
+ break;
+ case FileReader::NotFound:
+ return true;
+ default:
+ *err = load_err;
+ return false;
+ }
+
+ bool valid_header = false;
int version = 0;
- if (!fgets(buf, sizeof(buf), f) || fread(&version, 4, 1, f) < 1)
- valid_header = false;
+ if (log->file->content().size() >= kFileHeaderSize ||
+ log->file->content().substr(0, kFileSignature.size()) == kFileSignature) {
+ valid_header = true;
+ memcpy(&version,
+ log->file->content().data() + kFileSignature.size(),
+ sizeof(version));
+ }
+
// Note: For version differences, this should migrate to the new format.
// But the v1 format could sometimes (rarely) end up with invalid data, so
// don't migrate v1 to v3 to force a rebuild. (v2 only existed for a few days,
// and there was no release with it, so pretend that it never happened.)
- if (!valid_header || strcmp(buf, kFileSignature) != 0 ||
- version != kCurrentVersion) {
+ if (!valid_header || version != kCurrentVersion) {
if (version == 1)
*err = "deps log version change; rebuilding";
else
*err = "bad deps log signature or version; starting over";
- fclose(f);
+ log->file.reset();
unlink(path.c_str());
// Don't report this as a failure. An empty deps log will cause
// us to rebuild the outputs anyway.
return true;
}
- long offset;
- bool read_failed = false;
- int unique_dep_record_count = 0;
- int total_dep_record_count = 0;
- for (;;) {
- offset = ftell(f);
+ log->data.words =
+ reinterpret_cast<const uint32_t*>(
+ log->file->content().data() + kFileHeaderSize);
+ log->data.size =
+ (log->file->content().size() - kFileHeaderSize) / sizeof(uint32_t);
- unsigned size;
- if (fread(&size, 4, 1, f) < 1) {
- if (!feof(f))
- read_failed = true;
- break;
- }
- bool is_deps = (size >> 31) != 0;
- size = size & 0x7FFFFFFF;
+ return true;
+}
- if (size > kMaxRecordSize || fread(buf, size, 1, f) < 1) {
- read_failed = true;
- break;
- }
+template <typename Func>
+static bool ForEachRecord(DepsLogData log, DepsLogWordSpan chunk,
+ Func&& callback) {
+ InputRecord record {};
+ for (size_t index = chunk.begin; index < chunk.end; index += record.size) {
+ record = ParseRecord(log, index);
+ if (!callback(static_cast<const InputRecord&>(record))) return false;
+ if (record.kind == InputRecord::InvalidHeader) break;
+ }
+ return true;
+}
- if (is_deps) {
- assert(size % 4 == 0);
- int* deps_data = reinterpret_cast<int*>(buf);
- int out_id = deps_data[0];
- TimeStamp mtime;
- mtime = (TimeStamp)(((uint64_t)(unsigned int)deps_data[2] << 32) |
- (uint64_t)(unsigned int)deps_data[1]);
- deps_data += 3;
- int deps_count = (size / 4) - 3;
+struct DepsLogNodeSpan {
+ int begin; // starting node ID of span
+ int end; // stopping node ID of span
+};
- Deps* deps = new Deps(mtime, deps_count);
- for (int i = 0; i < deps_count; ++i) {
- assert(deps_data[i] < (int)nodes_.size());
- assert(nodes_[deps_data[i]]);
- deps->nodes[i] = nodes_[deps_data[i]];
+/// Determine the range of node IDs for each chunk of the deps log. The range
+/// for a chunk will only be used if the preceding chunks are valid.
+static std::vector<DepsLogNodeSpan>
+FindInitialNodeSpans(DepsLogData log,
+ const std::vector<DepsLogWordSpan>& chunk_words,
+ ThreadPool* thread_pool) {
+ // First count the number of path records (nodes) in each chunk.
+ std::vector<int> chunk_node_counts = ParallelMap(thread_pool, chunk_words,
+ [log](DepsLogWordSpan chunk) {
+ int node_count = 0;
+ ForEachRecord(log, chunk, [&node_count](const InputRecord& record) {
+ if (record.kind == InputRecord::PathRecord) {
+ ++node_count;
}
+ return true;
+ });
+ return node_count;
+ });
- total_dep_record_count++;
- if (!UpdateDeps(out_id, deps))
- ++unique_dep_record_count;
+ // Compute an initial [begin, end) node ID range for each chunk.
+ std::vector<DepsLogNodeSpan> result;
+ int next_node_id = 0;
+ for (int num_nodes : chunk_node_counts) {
+ result.push_back({ next_node_id, next_node_id + num_nodes });
+ next_node_id += num_nodes;
+ }
+ return result;
+}
+
+static bool IsValidRecord(const InputRecord& record, int next_node_id) {
+ auto is_valid_id = [next_node_id](int id) {
+ return id >= 0 && id < next_node_id;
+ };
+
+ switch (record.kind) {
+ case InputRecord::InvalidHeader:
+ return false;
+ case InputRecord::PathRecord:
+ // Validate the path's checksum.
+ if (record.u.path.checksum != ~next_node_id) {
+ return false;
+ }
+ break;
+ case InputRecord::DepsRecord:
+ // Verify that input/output node IDs are valid.
+ if (!is_valid_id(record.u.deps.output_id)) {
+ return false;
+ }
+ for (size_t i = 0; i < record.u.deps.deps_count; ++i) {
+ if (!is_valid_id(record.u.deps.deps[i])) {
+ return false;
+ }
+ }
+ break;
+ }
+ return true;
+}
+
+/// Validate the deps log. If there is an invalid record, the function truncates
+/// the word+node span vectors just before the invalid record.
+static void ValidateDepsLog(DepsLogData log,
+ std::vector<DepsLogWordSpan>* chunk_words,
+ std::vector<DepsLogNodeSpan>* chunk_nodes,
+ ThreadPool* thread_pool) {
+ std::atomic<size_t> num_valid_chunks { chunk_words->size() };
+
+ ParallelMap(thread_pool, IntegralRange<size_t>(0, num_valid_chunks),
+ [&](size_t chunk_index) {
+
+ size_t next_word_idx = (*chunk_words)[chunk_index].begin;
+ int next_node_id = (*chunk_nodes)[chunk_index].begin;
+
+ bool success = ForEachRecord(log, (*chunk_words)[chunk_index],
+ [&](const InputRecord& record) {
+ if (!IsValidRecord(record, next_node_id)) {
+ return false;
+ }
+ next_word_idx += record.size;
+ if (record.kind == InputRecord::PathRecord) {
+ ++next_node_id;
+ }
+ return true;
+ });
+
+ if (success) {
+ assert(next_word_idx == (*chunk_words)[chunk_index].end);
+ assert(next_node_id == (*chunk_nodes)[chunk_index].end);
} else {
- int path_size = size - 4;
- assert(path_size > 0); // CanonicalizePath() rejects empty paths.
- // There can be up to 3 bytes of padding.
- if (buf[path_size - 1] == '\0') --path_size;
- if (buf[path_size - 1] == '\0') --path_size;
- if (buf[path_size - 1] == '\0') --path_size;
- StringPiece subpath(buf, path_size);
- // It is not necessary to pass in a correct slash_bits here. It will
- // either be a Node that's in the manifest (in which case it will already
- // have a correct slash_bits that GetNode will look up), or it is an
- // implicit dependency from a .d which does not affect the build command
- // (and so need not have its slashes maintained).
- Node* node = state->GetNode(subpath, 0);
+ (*chunk_words)[chunk_index].end = next_word_idx;
+ (*chunk_nodes)[chunk_index].end = next_node_id;
+ AtomicUpdateMinimum(&num_valid_chunks, chunk_index + 1);
+ }
+ });
- // Check that the expected index matches the actual index. This can only
- // happen if two ninja processes write to the same deps log concurrently.
- // (This uses unary complement to make the checksum look less like a
- // dependency record entry.)
- unsigned checksum = *reinterpret_cast<unsigned*>(buf + size - 4);
- int expected_id = ~checksum;
- int id = nodes_.size();
- if (id != expected_id) {
- read_failed = true;
- break;
+ chunk_words->resize(num_valid_chunks);
+ chunk_nodes->resize(num_valid_chunks);
+}
+
+bool DepsLog::Load(const string& path, State* state, string* err) {
+ METRIC_RECORD(".ninja_deps load");
+
+ assert(nodes_.empty());
+ DepsLogInputFile log_file;
+
+ if (!OpenDepsLogForReading(path, &log_file, err)) return false;
+ if (log_file.file.get() == nullptr) return true;
+
+ DepsLogData log = log_file.data;
+
+ std::unique_ptr<ThreadPool> thread_pool = CreateThreadPool();
+
+ std::vector<DepsLogWordSpan> chunk_words = SplitDepsLog(log, thread_pool.get());
+ std::vector<DepsLogNodeSpan> chunk_nodes =
+ FindInitialNodeSpans(log, chunk_words, thread_pool.get());
+
+ // Validate the log and truncate the vectors after an invalid record.
+ ValidateDepsLog(log, &chunk_words, &chunk_nodes, thread_pool.get());
+ assert(chunk_words.size() == chunk_nodes.size());
+
+ const size_t chunk_count = chunk_words.size();
+ const int node_count = chunk_nodes.empty() ? 0 : chunk_nodes.back().end;
+
+ // The state path hash table doesn't automatically resize, so make sure that
+ // it has at least one bucket for each node in this deps log.
+ state->paths_.reserve(node_count);
+
+ nodes_.resize(node_count);
+
+ // A map from a node ID to the final file index of the deps record outputting
+ // the given node ID.
+ std::vector<std::atomic<ssize_t>> dep_index(node_count);
+ for (auto& index : dep_index) {
+ // Write a value of -1 to indicate that no deps record outputs this ID. We
+ // don't need these stores to be synchronized with other threads, so use
+ // relaxed stores, which are much faster.
+ index.store(-1, std::memory_order_relaxed);
+ }
+
+ // Add the nodes into the build graph, find the last deps record
+ // outputting each node, and count the total number of deps records.
+ const std::vector<size_t> dep_record_counts = ParallelMap(thread_pool.get(),
+ IntegralRange<size_t>(0, chunk_count),
+ [log, state, chunk_words, chunk_nodes, &dep_index,
+ this](size_t chunk_index) {
+ size_t next_word_idx = chunk_words[chunk_index].begin;
+ int next_node_id = chunk_nodes[chunk_index].begin;
+ int stop_node_id = chunk_nodes[chunk_index].end;
+ (void)stop_node_id; // suppress unused variable compiler warning
+ size_t dep_record_count = 0;
+
+ ForEachRecord(log, chunk_words[chunk_index],
+ [&, this](const InputRecord& record) {
+ assert(record.kind != InputRecord::InvalidHeader);
+ if (record.kind == InputRecord::PathRecord) {
+ int node_id = next_node_id++;
+ assert(node_id < stop_node_id);
+ assert(record.u.path.checksum == ~node_id);
+
+ // It is not necessary to pass in a correct slash_bits here. It will
+ // either be a Node that's in the manifest (in which case it will
+ // already have a correct slash_bits that GetNode will look up), or it
+ // is an implicit dependency from a .d which does not affect the build
+ // command (and so need not have its slashes maintained).
+ Node* node = state->GetNode(record.u.path.path, 0);
+ assert(node->id() < 0);
+ node->set_id(node_id);
+ nodes_[node_id] = node;
+ } else if (record.kind == InputRecord::DepsRecord) {
+ const int output_id = record.u.deps.output_id;
+ assert(static_cast<size_t>(output_id) < dep_index.size());
+ AtomicUpdateMaximum(&dep_index[output_id],
+ static_cast<ssize_t>(next_word_idx));
+ ++dep_record_count;
}
+ next_word_idx += record.size;
+ return true;
+ });
+ assert(next_node_id == stop_node_id);
+ return dep_record_count;
+ });
- assert(node->id() < 0);
- node->set_id(id);
- nodes_.push_back(node);
+ // Count the number of total and unique deps records.
+ const size_t total_dep_record_count =
+ std::accumulate(dep_record_counts.begin(), dep_record_counts.end(),
+ static_cast<size_t>(0));
+ size_t unique_dep_record_count = 0;
+ for (auto& index : dep_index) {
+ if (index.load(std::memory_order_relaxed) != -1) {
+ ++unique_dep_record_count;
}
}
- if (read_failed) {
- // An error occurred while loading; try to recover by truncating the
- // file to the last fully-read record.
- if (ferror(f)) {
- *err = strerror(ferror(f));
- } else {
- *err = "premature end of file";
- }
- fclose(f);
+ // Add the deps records.
+ deps_.resize(node_count);
+ ParallelMap(thread_pool.get(), IntegralRange<int>(0, node_count),
+ [this, log, &dep_index](int node_id) {
+ ssize_t index = dep_index[node_id];
+ if (index == -1) return;
- if (!Truncate(path, offset, err))
+ InputRecord record = ParseRecord(log, index);
+ assert(record.kind == InputRecord::DepsRecord);
+ assert(record.u.deps.output_id == node_id);
+
+ Deps* deps = new Deps(record.u.deps.mtime, record.u.deps.deps_count);
+ for (size_t i = 0; i < record.u.deps.deps_count; ++i) {
+ const int input_id = record.u.deps.deps[i];
+ assert(static_cast<size_t>(input_id) < nodes_.size());
+ Node* node = nodes_[input_id];
+ assert(node != nullptr);
+ deps->nodes[i] = node;
+ }
+ deps_[node_id] = deps;
+ });
+
+ const size_t actual_file_size = log_file.file->content().size();
+ const size_t parsed_file_size = kFileHeaderSize +
+ (chunk_words.empty() ? 0 : chunk_words.back().end) * sizeof(uint32_t);
+ assert(parsed_file_size <= actual_file_size);
+ if (parsed_file_size < actual_file_size) {
+ // An error occurred while loading; try to recover by truncating the file to
+ // the last fully-read record.
+ *err = "premature end of file";
+ log_file.file.reset();
+
+ if (!Truncate(path, parsed_file_size, err))
return false;
// The truncate succeeded; we'll just report the load error as a
@@ -292,11 +657,9 @@
return true;
}
- fclose(f);
-
// Rebuild the log if there are too many dead records.
- int kMinCompactionEntryCount = 1000;
- int kCompactionRatio = 3;
+ const unsigned kMinCompactionEntryCount = 1000;
+ const unsigned kCompactionRatio = 3;
if (total_dep_record_count > kMinCompactionEntryCount &&
total_dep_record_count > unique_dep_record_count * kCompactionRatio) {
needs_recompaction_ = true;
@@ -313,7 +676,7 @@
return deps_[node->id()];
}
-bool DepsLog::Recompact(const string& path, string* err) {
+bool DepsLog::Recompact(const string& path, const DiskInterface& disk, string* err) {
METRIC_RECORD(".ninja_deps recompact");
Close();
@@ -324,7 +687,7 @@
unlink(temp_path.c_str());
DepsLog new_log;
- if (!new_log.OpenForWrite(temp_path, err))
+ if (!new_log.OpenForWrite(temp_path, disk, err))
return false;
// Clear all known ids so that new ones can be reassigned. The new indices
@@ -337,8 +700,21 @@
Deps* deps = deps_[old_id];
if (!deps) continue; // If nodes_[old_id] is a leaf, it has no deps.
- if (!IsDepsEntryLiveFor(nodes_[old_id]))
- continue;
+ Node* node = nodes_[old_id];
+ if (node->in_edge()) {
+ // If the current manifest defines this edge, skip if it's not dep
+ // producing.
+ if (node->in_edge()->GetBinding("deps").empty()) continue;
+ } else {
+ // If the current manifest does not define this edge, skip if it's missing
+ // from the disk.
+ string err;
+ TimeStamp mtime = disk.LStat(node->path(), nullptr, &err);
+ if (mtime == -1)
+ Error("%s", err.c_str()); // log and ignore LStat() errors
+ if (mtime == 0)
+ continue;
+ }
if (!new_log.RecordDeps(nodes_[old_id], deps->mtime,
deps->node_count, deps->nodes)) {
@@ -368,11 +744,8 @@
bool DepsLog::IsDepsEntryLiveFor(Node* node) {
// Skip entries that don't have in-edges or whose edges don't have a
- // "deps" attribute. They were in the deps log from previous builds, but
- // the the files they were for were removed from the build and their deps
- // entries are no longer needed.
- // (Without the check for "deps", a chain of two or more nodes that each
- // had deps wouldn't be collected in a single recompaction.)
+ // "deps" attribute. They were in the deps log from previous builds, but the
+ // files they were for were removed from the build.
return node->in_edge() && !node->in_edge()->GetBinding("deps").empty();
}
diff --git a/src/deps_log.h b/src/deps_log.h
index 3812a28..c4a60e2 100644
--- a/src/deps_log.h
+++ b/src/deps_log.h
@@ -23,9 +23,19 @@
#include "timestamp.h"
+struct DiskInterface;
struct Node;
struct State;
+/// A buffer of uint32 words used to load a deps log. Does not include the
+/// log file's header.
+struct DepsLogData {
+ const uint32_t* words = nullptr;
+ size_t size = 0;
+};
+
+size_t MustBeDepsRecordHeader(DepsLogData log, size_t index);
+
/// As build commands run they can output extra dependency information
/// (e.g. header dependencies for C source) dynamically. DepsLog collects
/// that information at build time and uses it for subsequent builds.
@@ -70,7 +80,7 @@
~DepsLog();
// Writing (build-time) interface.
- bool OpenForWrite(const string& path, string* err);
+ bool OpenForWrite(const string& path, const DiskInterface& disk, string* err);
bool RecordDeps(Node* node, TimeStamp mtime, const vector<Node*>& nodes);
bool RecordDeps(Node* node, TimeStamp mtime, int node_count, Node** nodes);
void Close();
@@ -88,7 +98,7 @@
Deps* GetDeps(Node* node);
/// Rewrite the known log entries, throwing away old data.
- bool Recompact(const string& path, string* err);
+ bool Recompact(const string& path, const DiskInterface& disk, string* err);
/// Returns if the deps entry for a node is still reachable from the manifest.
///
diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc
index 0cdeb45..c43529e 100644
--- a/src/deps_log_test.cc
+++ b/src/deps_log_test.cc
@@ -41,7 +41,8 @@
State state1;
DepsLog log1;
string err;
- EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log1.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
{
@@ -93,9 +94,14 @@
State state1;
DepsLog log1;
string err;
- EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log1.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
+ // The paths_ concurrent hash table doesn't automatically resize itself, so
+ // reserve space in advance before synthesizing paths.
+ state1.paths_.reserve(kNumDeps);
+
{
vector<Node*> deps;
for (int i = 0; i < kNumDeps; ++i) {
@@ -128,7 +134,8 @@
State state;
DepsLog log;
string err;
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -148,9 +155,10 @@
State state;
DepsLog log;
string err;
+ VirtualFileSystem fs;
EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -182,7 +190,8 @@
ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
DepsLog log;
string err;
- ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ ASSERT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -210,9 +219,10 @@
ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
DepsLog log;
string err;
+ VirtualFileSystem fs;
ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
- ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -252,7 +262,8 @@
ASSERT_EQ("foo.h", deps->nodes[0]->path());
ASSERT_EQ("baz.h", deps->nodes[1]->path());
- ASSERT_TRUE(log.Recompact(kTestFilename, &err));
+ VirtualFileSystem fs;
+ ASSERT_TRUE(log.Recompact(kTestFilename, fs, &err));
// The in-memory deps graph should still be valid after recompaction.
deps = log.GetDeps(out);
@@ -301,17 +312,25 @@
ASSERT_EQ("foo.h", deps->nodes[0]->path());
ASSERT_EQ("baz.h", deps->nodes[1]->path());
- ASSERT_TRUE(log.Recompact(kTestFilename, &err));
+ // Keep out.o dependencies.
+ VirtualFileSystem fs;
+ fs.Create("out.o", "");
+
+ ASSERT_TRUE(log.Recompact(kTestFilename, fs, &err));
+
+ deps = log.GetDeps(out);
+ ASSERT_TRUE(deps);
+ ASSERT_EQ(1, deps->mtime);
+ ASSERT_EQ(1, deps->node_count);
+ ASSERT_EQ("foo.h", deps->nodes[0]->path());
+ ASSERT_EQ(out, log.nodes()[out->id()]);
// The previous entries should have been removed.
- deps = log.GetDeps(out);
- ASSERT_FALSE(deps);
-
deps = log.GetDeps(other_out);
ASSERT_FALSE(deps);
+ //ASSERT_EQ(-1, state.LookupNode("foo.h")->id());
// The .h files pulled in via deps should no longer have ids either.
- ASSERT_EQ(-1, state.LookupNode("foo.h")->id());
ASSERT_EQ(-1, state.LookupNode("baz.h")->id());
// The file should have shrunk more.
@@ -355,7 +374,8 @@
State state;
DepsLog log;
string err;
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -414,7 +434,8 @@
State state;
DepsLog log;
string err;
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -450,7 +471,8 @@
// The truncated entry should've been discarded.
EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o", 0)));
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
// Add a new entry.
@@ -476,4 +498,203 @@
}
}
+template <typename Func>
+static void DoLoadInvalidLogTest(Func&& func) {
+ State state;
+ DepsLog log;
+ std::string err;
+ ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
+ ASSERT_EQ("premature end of file; recovering", err);
+ func(&state, &log);
+}
+
+TEST_F(DepsLogTest, LoadInvalidLog) {
+ struct Item {
+ Item(int num) : is_num(true), num(num) {}
+ Item(const char* str) : is_num(false), str(str) {}
+
+ bool is_num;
+ uint32_t num;
+ const char* str;
+ };
+
+ auto write_file = [](std::vector<Item> items) {
+ FILE* fp = fopen(kTestFilename, "wb");
+ for (const Item& item : items) {
+ if (item.is_num) {
+ ASSERT_EQ(1, fwrite(&item.num, sizeof(item.num), 1, fp));
+ } else {
+ ASSERT_EQ(strlen(item.str), fwrite(item.str, 1, strlen(item.str), fp));
+ }
+ }
+ fclose(fp);
+ };
+
+ const int kCurrentVersion = 4;
+ auto path_hdr = [](int path_len) -> int {
+ return RoundUp(path_len, 4) + 4;
+ };
+ auto deps_hdr = [](int deps_cnt) -> int {
+ return 0x80000000 | ((3 * sizeof(uint32_t)) + (deps_cnt * 4));
+ };
+
+ write_file({
+ "# ninjadeps\n", kCurrentVersion,
+ path_hdr(4), "foo0", ~0, // node #0
+ path_hdr(4), "foo1", ~2, // invalid path ID
+ });
+ DoLoadInvalidLogTest([](State* state, DepsLog* log) {
+ ASSERT_EQ(0, state->LookupNode("foo0")->id());
+ ASSERT_EQ(nullptr, state->LookupNode("foo1"));
+ });
+
+ write_file({
+ "# ninjadeps\n", kCurrentVersion,
+ path_hdr(4), "foo0", ~0, // node #0
+ deps_hdr(1), /*node*/0, /*mtime*/5, 0, /*node*/1, // invalid src ID
+ path_hdr(4), "foo1", ~1, // node #1
+ });
+ DoLoadInvalidLogTest([](State* state, DepsLog* log) {
+ ASSERT_EQ(0, state->LookupNode("foo0")->id());
+ ASSERT_EQ(nullptr, log->GetDeps(state->LookupNode("foo0")));
+ ASSERT_EQ(nullptr, state->LookupNode("foo1"));
+ });
+
+ write_file({
+ "# ninjadeps\n", kCurrentVersion,
+ path_hdr(4), "foo0", ~0, // node #0
+ deps_hdr(1), /*node*/1, /*mtime*/5, 0, /*node*/0, // invalid out ID
+ path_hdr(4), "foo1", ~1, // node #1
+ });
+ DoLoadInvalidLogTest([](State* state, DepsLog* log) {
+ ASSERT_EQ(0, state->LookupNode("foo0")->id());
+ ASSERT_EQ(nullptr, state->LookupNode("foo1"));
+ });
+
+ write_file({
+ "# ninjadeps\n", kCurrentVersion,
+ path_hdr(4), "foo0", ~0, // node #0
+ path_hdr(4), "foo1", ~1, // node #1
+ path_hdr(4), "foo2", ~2, // node #2
+ deps_hdr(1), /*node*/2, /*mtime*/5, 0, /*node*/1,
+ deps_hdr(1), /*node*/2, /*mtime*/6, 0, /*node*/3, // invalid src ID
+
+ // No records after the invalid record are parsed.
+ path_hdr(4), "foo3", ~3, // node #3
+ deps_hdr(1), /*node*/3, /*mtime*/7, 0, /*node*/0,
+ path_hdr(4), "foo4", ~4, // node #4
+ deps_hdr(1), /*node*/4, /*mtime*/8, 0, /*node*/0,
+
+ // Truncation must be handled before looking for the last deps record
+ // that outputs a given node.
+ deps_hdr(1), /*node*/2, /*mtime*/9, 0, /*node*/0,
+ deps_hdr(1), /*node*/2, /*mtime*/9, 0, /*node*/3,
+ });
+ DoLoadInvalidLogTest([](State* state, DepsLog* log) {
+ ASSERT_EQ(0, state->LookupNode("foo0")->id());
+ ASSERT_EQ(1, state->LookupNode("foo1")->id());
+ ASSERT_EQ(2, state->LookupNode("foo2")->id());
+ ASSERT_EQ(nullptr, state->LookupNode("foo3"));
+ ASSERT_EQ(nullptr, state->LookupNode("foo4"));
+
+ ASSERT_EQ(nullptr, log->GetDeps(state->LookupNode("foo1")));
+
+ DepsLog::Deps* deps = log->GetDeps(state->LookupNode("foo2"));
+ ASSERT_EQ(5, deps->mtime);
+ ASSERT_EQ(1, deps->node_count);
+ ASSERT_EQ(1, deps->nodes[0]->id());
+ });
+}
+
+TEST_F(DepsLogTest, MustBeDepsRecordHeader) {
+ // Mark a word as a candidate.
+ static constexpr uint64_t kCandidate = 0x100000000;
+
+ // Verifies that MustBeDepsRecordHeader returns the expected value. Returns
+ // true on success.
+ auto do_test = [](std::vector<uint64_t> words) -> bool {
+ std::vector<uint32_t> data;
+ for (uint64_t word : words) {
+ // Coerce from uint64_t to uint32_t to mask off the kCandidate flag.
+ data.push_back(word);
+ }
+ DepsLogData log;
+ log.words = data.data();
+ log.size = data.size();
+ for (size_t i = 0; i < words.size(); ++i) {
+ const bool expected = (words[i] & kCandidate) == kCandidate;
+ if (expected != MustBeDepsRecordHeader(log, i)) {
+ printf("\n%s,%d: bad index: %zu\n", __FILE__, __LINE__, i);
+ return false;
+ }
+ }
+ return true;
+ };
+
+ // Two valid deps records with no dependencies. Each record's header is
+ // recognized as the start of a deps record. The first record has an mtime_hi
+ // from 2262.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x8000000c|kCandidate, 1, 2, 0x80000100,
+ 0x8000000c|kCandidate, 3, 4, 5,
+ }));
+
+ // The first record's mtime_lo is within a 524us window. The second record's
+ // header looks like a potential mtime_lo for 0x8007fffc.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x8000000c|kCandidate, 1, 0x8007fffc, 2,
+ 0x8000000c, 3, 4, 5,
+ }));
+
+ // 0x80080000 is above the maximum record size, so it is rejected as a
+ // possible header.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x8000000c|kCandidate, 1, 0x80080000, 2,
+ 0x8000000c|kCandidate, 3, 4, 5,
+ }));
+
+ // Two deps records with >16K inputs each. The header could be confused with a
+ // path string containing control characters, so it's not a candidate.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x80010101, 1, 2, 3, // input IDs elided...
+ 0x80010101, 4, 5, 6, // input IDs elided...
+ }));
+
+ // The first record has a single dependency and an mtime_hi from 2262. The
+ // second deps record's header looks like a potential mtime_lo for 0x80000100.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x80000010|kCandidate, 1, 2, 0x80000100, 3,
+ 0x8000000c, 4, 5, 6,
+ }));
+
+ // The first deps record's mtime_lo is within a 524us window, and the second
+ // record's header looks like a potential mtime_hi for 0x80000100.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x80000010|kCandidate, 1, 0x80000100, 2, 3,
+ 0x8000000c, 4, 5, 6,
+ }));
+
+ // The first record has two dependencies, so its mtime_lo doesn't disqualify
+ // the next record's header.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x80000014|kCandidate, 1, 0x80000100, 2, 3, 4,
+ 0x8000000c|kCandidate, 5, 6, 7,
+ }));
+
+ // The first deps record's mtime_hi is from 2262, and the second record's
+ // header looks like a potential mtime_hi for 0x80000100.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x80000014|kCandidate, 1, 2, 0x80000100, 3, 4,
+ 0x8000000c, 5, 6, 7,
+ }));
+}
+
} // anonymous namespace
diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index d4c2fb0..0044c5b 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -153,6 +153,34 @@
// RealDiskInterface -----------------------------------------------------------
+#ifndef _WIN32
+static TimeStamp StatTimestamp(const struct stat& st) {
+ // Some users (Flatpak) set mtime to 0, this should be harmless
+ // and avoids conflicting with our return value of 0 meaning
+ // that it doesn't exist.
+ if (st.st_mtime == 0)
+ return 1;
+#if defined(__APPLE__) && !defined(_POSIX_C_SOURCE)
+ return ((int64_t)st.st_mtimespec.tv_sec * 1000000000LL +
+ st.st_mtimespec.tv_nsec);
+#elif (_POSIX_C_SOURCE >= 200809L || _XOPEN_SOURCE >= 700 || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \
+ defined(__BIONIC__) || (defined (__SVR4) && defined (__sun)) || defined(__FreeBSD__))
+ // For glibc, see "Timestamp files" in the Notes of http://www.kernel.org/doc/man-pages/online/pages/man2/stat.2.html
+ // newlib, uClibc and musl follow the kernel (or Cygwin) headers and define the right macro values above.
+ // For bsd, see https://github.com/freebsd/freebsd/blob/master/sys/sys/stat.h and similar
+ // For bionic, C and POSIX API is always enabled.
+ // For solaris, see https://docs.oracle.com/cd/E88353_01/html/E37841/stat-2.html.
+ return (int64_t)st.st_mtim.tv_sec * 1000000000LL + st.st_mtim.tv_nsec;
+#elif defined(_AIX)
+ return (int64_t)st.st_mtime * 1000000000LL + st.st_mtime_n;
+#else
+ return (int64_t)st.st_mtime * 1000000000LL + st.st_mtimensec;
+#endif
+}
+#endif
+
+/// This function is thread-safe on Unix but not on Windows. See
+/// RealDiskInterface::IsStatThreadSafe.
TimeStamp RealDiskInterface::Stat(const string& path, string* err) const {
METRIC_RECORD("node stat");
#ifdef _WIN32
@@ -197,27 +225,34 @@
*err = "stat(" + path + "): " + strerror(errno);
return -1;
}
- // Some users (Flatpak) set mtime to 0, this should be harmless
- // and avoids conflicting with our return value of 0 meaning
- // that it doesn't exist.
- if (st.st_mtime == 0)
- return 1;
-#if defined(__APPLE__) && !defined(_POSIX_C_SOURCE)
- return ((int64_t)st.st_mtimespec.tv_sec * 1000000000LL +
- st.st_mtimespec.tv_nsec);
-#elif (_POSIX_C_SOURCE >= 200809L || _XOPEN_SOURCE >= 700 || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \
- defined(__BIONIC__) || (defined (__SVR4) && defined (__sun)) || defined(__FreeBSD__))
- // For glibc, see "Timestamp files" in the Notes of http://www.kernel.org/doc/man-pages/online/pages/man2/stat.2.html
- // newlib, uClibc and musl follow the kernel (or Cygwin) headers and define the right macro values above.
- // For bsd, see https://github.com/freebsd/freebsd/blob/master/sys/sys/stat.h and similar
- // For bionic, C and POSIX API is always enabled.
- // For solaris, see https://docs.oracle.com/cd/E88353_01/html/E37841/stat-2.html.
- return (int64_t)st.st_mtim.tv_sec * 1000000000LL + st.st_mtim.tv_nsec;
-#elif defined(_AIX)
- return (int64_t)st.st_mtime * 1000000000LL + st.st_mtime_n;
-#else
- return (int64_t)st.st_mtime * 1000000000LL + st.st_mtimensec;
+ return StatTimestamp(st);
#endif
+}
+
+TimeStamp RealDiskInterface::LStat(const string& path, bool* is_dir, string* err) const {
+ METRIC_RECORD("node lstat");
+#ifdef _WIN32
+#error unimplemented
+#else
+ struct stat st;
+ if (lstat(path.c_str(), &st) < 0) {
+ if (errno == ENOENT || errno == ENOTDIR)
+ return 0;
+ *err = "lstat(" + path + "): " + strerror(errno);
+ return -1;
+ }
+ if (is_dir != nullptr) {
+ *is_dir = S_ISDIR(st.st_mode);
+ }
+ return StatTimestamp(st);
+#endif
+}
+
+bool RealDiskInterface::IsStatThreadSafe() const {
+#ifdef _WIN32
+ return false;
+#else
+ return true;
#endif
}
@@ -266,6 +301,35 @@
}
}
+FileReader::Status RealDiskInterface::LoadFile(const std::string& path,
+ std::unique_ptr<LoadedFile>* result,
+ std::string* err) {
+#ifdef _WIN32
+#error "LoadFile is not implemented yet on Windows"
+#else
+ struct MappedFile : LoadedFile {
+ MappedFile(const std::string& filename, std::unique_ptr<Mapping> mapping)
+ : mapping_(std::move(mapping)) {
+ filename_ = filename;
+ content_with_nul_ = { mapping_->base(), mapping_->file_size() + 1 };
+ }
+ std::unique_ptr<Mapping> mapping_;
+ };
+
+ std::unique_ptr<Mapping> mapping;
+ switch (::MapFile(path, &mapping, err)) {
+ case 0:
+ *result = std::unique_ptr<MappedFile>(new MappedFile(path,
+ std::move(mapping)));
+ return Okay;
+ case -ENOENT:
+ return NotFound;
+ default:
+ return OtherError;
+ }
+#endif
+}
+
int RealDiskInterface::RemoveFile(const string& path) {
if (remove(path.c_str()) < 0) {
switch (errno) {
diff --git a/src/disk_interface.h b/src/disk_interface.h
index 145e089..f8b1b0d 100644
--- a/src/disk_interface.h
+++ b/src/disk_interface.h
@@ -16,11 +16,64 @@
#define NINJA_DISK_INTERFACE_H_
#include <map>
+#include <memory>
#include <string>
using namespace std;
+#include "string_piece.h"
#include "timestamp.h"
+/// A file whose content has been loaded into memory.
+struct LoadedFile {
+ LoadedFile() {}
+
+ LoadedFile(const LoadedFile&) = delete;
+ LoadedFile& operator=(const LoadedFile&) = delete;
+
+ virtual ~LoadedFile() {}
+
+ const std::string& filename() const { return filename_; }
+
+ /// Return the size of the file's content, excluding the extra NUL at the end.
+ size_t size() const { return content_with_nul_.size() - 1; }
+
+ /// Return the file content, excluding the extra NUL at the end.
+ StringPiece content() const {
+ return content_with_nul_.substr(0, content_with_nul_.size() - 1);
+ }
+
+ /// The last character of this StringPiece will be NUL. (i.e. The size of this
+ /// view will be 1 greater than the true file size.)
+ StringPiece content_with_nul() const { return content_with_nul_; }
+
+protected:
+ std::string filename_;
+
+ /// The start of the string view must be aligned sufficiently for any basic
+ /// type it may contain (e.g. a minimum alignment of 16 bytes is typical). The
+ /// alignment minimum is necessary for parsing the 4-byte aligned binary
+ /// .ninja_deps log.
+ StringPiece content_with_nul_;
+};
+
+/// An implementation of LoadedFile where the file's content is copied into a
+/// char[] array. This class adds a NUL to the given string.
+struct HeapLoadedFile : LoadedFile {
+ HeapLoadedFile(const std::string& filename, StringPiece content) {
+ filename_ = filename;
+ // An array allocated with new char[N] is guaranteed to be sufficiently
+ // aligned for any fundamental type.
+ storage_.reset(new char[content.size() + 1]);
+ memcpy(storage_.get(), content.data(), content.size());
+ storage_[content.size()] = '\0';
+ content_with_nul_ = StringPiece(storage_.get(), content.size() + 1);
+ }
+
+private:
+ /// Use new char[] to provide the alignment guarantee this class needs.
+ std::unique_ptr<char[]> storage_;
+};
+
/// Interface for reading files from disk. See DiskInterface for details.
/// This base offers the minimum interface needed just to read files.
struct FileReader {
@@ -37,6 +90,12 @@
/// On error, return another Status and fill |err|.
virtual Status ReadFile(const string& path, string* contents,
string* err) = 0;
+
+ /// Open the file for reading and return an abstract LoadedFile. On success,
+ /// return Okay. On error, return another Status and fill |err|.
+ virtual Status LoadFile(const std::string& path,
+ std::unique_ptr<LoadedFile>* result,
+ std::string* err) = 0;
};
/// Interface for accessing the disk.
@@ -45,9 +104,18 @@
/// is RealDiskInterface.
struct DiskInterface: public FileReader {
/// stat() a file, returning the mtime, or 0 if missing and -1 on
- /// other errors.
+ /// other errors. Thread-safe iff IsStatThreadSafe returns true.
virtual TimeStamp Stat(const string& path, string* err) const = 0;
+ /// lstat() a path, returning the mtime, or 0 if missing and 01 on
+ /// other errors. Does not traverse symlinks, and returns whether the
+ /// path represents a directory. Thread-safe iff IsStatThreadSafe
+ /// returns true.
+ virtual TimeStamp LStat(const string& path, bool* is_dir, string* err) const = 0;
+
+ /// True if Stat() can be called from multiple threads concurrently.
+ virtual bool IsStatThreadSafe() const = 0;
+
/// Create a directory, returning false on failure.
virtual bool MakeDir(const string& path) = 0;
@@ -76,9 +144,14 @@
{}
virtual ~RealDiskInterface() {}
virtual TimeStamp Stat(const string& path, string* err) const;
+ virtual TimeStamp LStat(const string& path, bool* is_dir, string* err) const;
+ virtual bool IsStatThreadSafe() const;
virtual bool MakeDir(const string& path);
virtual bool WriteFile(const string& path, const string& contents);
virtual Status ReadFile(const string& path, string* contents, string* err);
+ virtual Status LoadFile(const std::string& path,
+ std::unique_ptr<LoadedFile>* result,
+ std::string* err);
virtual int RemoveFile(const string& path);
/// Whether stat information can be cached. Only has an effect on Windows.
diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc
index bac515d..090adc4 100644
--- a/src/disk_interface_test.cc
+++ b/src/disk_interface_test.cc
@@ -213,10 +213,12 @@
struct StatTest : public StateTestWithBuiltinRules,
public DiskInterface {
- StatTest() : scan_(&state_, NULL, NULL, this, NULL) {}
+ StatTest() : scan_(&state_, NULL, NULL, this, NULL, false) {}
// DiskInterface implementation.
virtual TimeStamp Stat(const string& path, string* err) const;
+ virtual TimeStamp LStat(const string& path, bool* is_dir, string* err) const;
+ virtual bool IsStatThreadSafe() const;
virtual bool WriteFile(const string& path, const string& contents) {
assert(false);
return true;
@@ -229,6 +231,12 @@
assert(false);
return NotFound;
}
+ virtual Status LoadFile(const std::string& path,
+ std::unique_ptr<LoadedFile>* result,
+ std::string* err) {
+ assert(false);
+ return NotFound;
+ }
virtual int RemoveFile(const string& path) {
assert(false);
return 0;
@@ -240,13 +248,23 @@
};
TimeStamp StatTest::Stat(const string& path, string* err) const {
+ return LStat(path, nullptr, err);
+}
+
+TimeStamp StatTest::LStat(const string& path, bool* is_dir, string* err) const {
stats_.push_back(path);
map<string, TimeStamp>::const_iterator i = mtimes_.find(path);
if (i == mtimes_.end())
return 0; // File not found.
+ if (is_dir != nullptr)
+ *is_dir = false;
return i->second;
}
+bool StatTest::IsStatThreadSafe() const {
+ return false;
+}
+
TEST_F(StatTest, Simple) {
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build out: cat in\n"));
@@ -256,7 +274,7 @@
EXPECT_TRUE(out->Stat(this, &err));
EXPECT_EQ("", err);
ASSERT_EQ(1u, stats_.size());
- scan_.RecomputeDirty(out, NULL);
+ scan_.RecomputeDirty(out, NULL, NULL);
ASSERT_EQ(2u, stats_.size());
ASSERT_EQ("out", stats_[0]);
ASSERT_EQ("in", stats_[1]);
@@ -272,7 +290,7 @@
EXPECT_TRUE(out->Stat(this, &err));
EXPECT_EQ("", err);
ASSERT_EQ(1u, stats_.size());
- scan_.RecomputeDirty(out, NULL);
+ scan_.RecomputeDirty(out, NULL, NULL);
ASSERT_EQ(3u, stats_.size());
ASSERT_EQ("out", stats_[0]);
ASSERT_TRUE(GetNode("out")->dirty());
@@ -292,7 +310,7 @@
EXPECT_TRUE(out->Stat(this, &err));
EXPECT_EQ("", err);
ASSERT_EQ(1u, stats_.size());
- scan_.RecomputeDirty(out, NULL);
+ scan_.RecomputeDirty(out, NULL, NULL);
ASSERT_EQ(1u + 6u, stats_.size());
ASSERT_EQ("mid1", stats_[1]);
ASSERT_TRUE(GetNode("mid1")->dirty());
@@ -313,7 +331,7 @@
EXPECT_TRUE(out->Stat(this, &err));
EXPECT_EQ("", err);
ASSERT_EQ(1u, stats_.size());
- scan_.RecomputeDirty(out, NULL);
+ scan_.RecomputeDirty(out, NULL, NULL);
ASSERT_FALSE(GetNode("in")->dirty());
ASSERT_TRUE(GetNode("mid")->dirty());
ASSERT_TRUE(GetNode("out")->dirty());
diff --git a/src/dyndep.cc b/src/dyndep.cc
index 2aee601..ff79395 100644
--- a/src/dyndep.cc
+++ b/src/dyndep.cc
@@ -40,7 +40,7 @@
return false;
// Update each edge that specified this node as its dyndep binding.
- std::vector<Edge*> const& out_edges = node->out_edges();
+ std::vector<Edge*> const& out_edges = node->GetOutEdges();
for (std::vector<Edge*>::const_iterator oe = out_edges.begin();
oe != out_edges.end(); ++oe) {
Edge* const edge = *oe;
@@ -83,7 +83,7 @@
// We know the edge already has its own binding
// scope because it has a "dyndep" binding.
if (dyndeps->restat_)
- edge->env_->AddBinding("restat", "1");
+ edge->SetRestat();
// Add the dyndep-discovered outputs to the edge.
edge->outputs_.insert(edge->outputs_.end(),
diff --git a/src/dyndep_parser.cc b/src/dyndep_parser.cc
index baebbac..ce3b03e 100644
--- a/src/dyndep_parser.cc
+++ b/src/dyndep_parser.cc
@@ -30,191 +30,201 @@
bool DyndepParser::Parse(const string& filename, const string& input,
string* err) {
- lexer_.Start(filename, input);
+ Lexer lexer(filename, input, input.data());
// Require a supported ninja_dyndep_version value immediately so
// we can exit before encountering any syntactic surprises.
bool haveDyndepVersion = false;
for (;;) {
- Lexer::Token token = lexer_.ReadToken();
+ Lexer::Token token = lexer.ReadToken();
switch (token) {
case Lexer::BUILD: {
if (!haveDyndepVersion)
- return lexer_.Error("expected 'ninja_dyndep_version = ...'", err);
- if (!ParseEdge(err))
+ return lexer.Error("expected 'ninja_dyndep_version = ...'", err);
+ if (!ParseEdge(lexer, err))
return false;
break;
}
case Lexer::IDENT: {
- lexer_.UnreadToken();
+ lexer.UnreadToken();
if (haveDyndepVersion)
- return lexer_.Error(string("unexpected ") + Lexer::TokenName(token),
+ return lexer.Error(string("unexpected ") + Lexer::TokenName(token),
err);
- if (!ParseDyndepVersion(err))
+ if (!ParseDyndepVersion(lexer, err))
return false;
haveDyndepVersion = true;
break;
}
case Lexer::ERROR:
- return lexer_.Error(lexer_.DescribeLastError(), err);
+ return lexer.Error(lexer.DescribeLastError(), err);
case Lexer::TEOF:
if (!haveDyndepVersion)
- return lexer_.Error("expected 'ninja_dyndep_version = ...'", err);
+ return lexer.Error("expected 'ninja_dyndep_version = ...'", err);
return true;
case Lexer::NEWLINE:
break;
default:
- return lexer_.Error(string("unexpected ") + Lexer::TokenName(token),
+ return lexer.Error(string("unexpected ") + Lexer::TokenName(token),
err);
}
}
return false; // not reached
}
-bool DyndepParser::ParseDyndepVersion(string* err) {
- string name;
- EvalString let_value;
- if (!ParseLet(&name, &let_value, err))
+static const HashedStrView kNinjaDyndepVersion { "ninja_dyndep_version" };
+
+bool DyndepParser::ParseDyndepVersion(Lexer& lexer, string* err) {
+ StringPiece name;
+ StringPiece let_value;
+ if (!ParseLet(lexer, &name, &let_value, err))
return false;
- if (name != "ninja_dyndep_version") {
- return lexer_.Error("expected 'ninja_dyndep_version = ...'", err);
+ if (name != kNinjaDyndepVersion) {
+ return lexer.Error("expected 'ninja_dyndep_version = ...'", err);
}
- string version = let_value.Evaluate(&env_);
+ string version;
+ EvaluateBindingInScope(&version, let_value, scope_);
int major, minor;
ParseVersion(version, &major, &minor);
if (major != 1 || minor != 0) {
- return lexer_.Error(
+ return lexer.Error(
string("unsupported 'ninja_dyndep_version = ") + version + "'", err);
return false;
}
return true;
}
-bool DyndepParser::ParseLet(string* key, EvalString* value, string* err) {
- if (!lexer_.ReadIdent(key))
- return lexer_.Error("expected variable name", err);
- if (!ExpectToken(Lexer::EQUALS, err))
+bool DyndepParser::ParseLet(Lexer& lexer, StringPiece* key, StringPiece* value, string* err) {
+ if (!lexer.ReadIdent(key))
+ return lexer.Error("expected variable name", err);
+ if (!ExpectToken(lexer, Lexer::EQUALS, err))
return false;
- if (!lexer_.ReadVarValue(value, err))
+ if (!lexer.ReadBindingValue(value, err))
return false;
return true;
}
-bool DyndepParser::ParseEdge(string* err) {
+static const HashedStrView kDyndep { "dyndep" };
+static const HashedStrView kRestat { "restat" };
+
+bool DyndepParser::ParseEdge(Lexer& lexer, string* err) {
// Parse one explicit output. We expect it to already have an edge.
// We will record its dynamically-discovered dependency information.
Dyndeps* dyndeps = NULL;
{
- EvalString out0;
- if (!lexer_.ReadPath(&out0, err))
+ LexedPath out0;
+ if (!lexer.ReadPath(&out0, err))
return false;
- if (out0.empty())
- return lexer_.Error("expected path", err);
+ if (out0.str_.empty())
+ return lexer.Error("expected path", err);
- string path = out0.Evaluate(&env_);
+ string path;
+ EvaluatePathInScope(&path, out0, scope_);
string path_err;
uint64_t slash_bits;
if (!CanonicalizePath(&path, &slash_bits, &path_err))
- return lexer_.Error(path_err, err);
+ return lexer.Error(path_err, err);
Node* node = state_->LookupNode(path);
if (!node || !node->in_edge())
- return lexer_.Error("no build statement exists for '" + path + "'", err);
+ return lexer.Error("no build statement exists for '" + path + "'", err);
Edge* edge = node->in_edge();
std::pair<DyndepFile::iterator, bool> res =
dyndep_file_->insert(DyndepFile::value_type(edge, Dyndeps()));
if (!res.second)
- return lexer_.Error("multiple statements for '" + path + "'", err);
+ return lexer.Error("multiple statements for '" + path + "'", err);
dyndeps = &res.first->second;
}
// Disallow explicit outputs.
{
- EvalString out;
- if (!lexer_.ReadPath(&out, err))
+ LexedPath out;
+ if (!lexer.ReadPath(&out, err))
return false;
- if (!out.empty())
- return lexer_.Error("explicit outputs not supported", err);
+ if (!out.str_.empty())
+ return lexer.Error("explicit outputs not supported", err);
}
// Parse implicit outputs, if any.
- vector<EvalString> outs;
- if (lexer_.PeekToken(Lexer::PIPE)) {
+ vector<LexedPath> outs;
+ if (lexer.PeekToken(Lexer::PIPE)) {
for (;;) {
- EvalString out;
- if (!lexer_.ReadPath(&out, err))
+ LexedPath out;
+ if (!lexer.ReadPath(&out, err))
return err;
- if (out.empty())
+ if (out.str_.empty())
break;
outs.push_back(out);
}
}
- if (!ExpectToken(Lexer::COLON, err))
+ if (!ExpectToken(lexer, Lexer::COLON, err))
return false;
- string rule_name;
- if (!lexer_.ReadIdent(&rule_name) || rule_name != "dyndep")
- return lexer_.Error("expected build command name 'dyndep'", err);
+ StringPiece rule_name;
+ if (!lexer.ReadIdent(&rule_name) || rule_name != kDyndep)
+ return lexer.Error("expected build command name 'dyndep'", err);
// Disallow explicit inputs.
{
- EvalString in;
- if (!lexer_.ReadPath(&in, err))
+ LexedPath in;
+ if (!lexer.ReadPath(&in, err))
return false;
- if (!in.empty())
- return lexer_.Error("explicit inputs not supported", err);
+ if (!in.str_.empty())
+ return lexer.Error("explicit inputs not supported", err);
}
// Parse implicit inputs, if any.
- vector<EvalString> ins;
- if (lexer_.PeekToken(Lexer::PIPE)) {
+ vector<LexedPath> ins;
+ if (lexer.PeekToken(Lexer::PIPE)) {
for (;;) {
- EvalString in;
- if (!lexer_.ReadPath(&in, err))
+ LexedPath in;
+ if (!lexer.ReadPath(&in, err))
return err;
- if (in.empty())
+ if (in.str_.empty())
break;
ins.push_back(in);
}
}
// Disallow order-only inputs.
- if (lexer_.PeekToken(Lexer::PIPE2))
- return lexer_.Error("order-only inputs not supported", err);
+ if (lexer.PeekToken(Lexer::PIPE2))
+ return lexer.Error("order-only inputs not supported", err);
- if (!ExpectToken(Lexer::NEWLINE, err))
+ if (!ExpectToken(lexer, Lexer::NEWLINE, err))
return false;
- if (lexer_.PeekToken(Lexer::INDENT)) {
- string key;
- EvalString val;
- if (!ParseLet(&key, &val, err))
+ if (lexer.PeekToken(Lexer::INDENT)) {
+ StringPiece key;
+ StringPiece val;
+ if (!ParseLet(lexer, &key, &val, err))
return false;
- if (key != "restat")
- return lexer_.Error("binding is not 'restat'", err);
- string value = val.Evaluate(&env_);
+ if (key != kRestat)
+ return lexer.Error("binding is not 'restat'", err);
+ string value;
+ EvaluateBindingInScope(&value, val, scope_);
dyndeps->restat_ = !value.empty();
}
dyndeps->implicit_inputs_.reserve(ins.size());
- for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) {
- string path = i->Evaluate(&env_);
+ for (vector<LexedPath>::iterator i = ins.begin(); i != ins.end(); ++i) {
+ string path;
+ EvaluatePathInScope(&path, *i, scope_);
string path_err;
uint64_t slash_bits;
if (!CanonicalizePath(&path, &slash_bits, &path_err))
- return lexer_.Error(path_err, err);
+ return lexer.Error(path_err, err);
Node* n = state_->GetNode(path, slash_bits);
dyndeps->implicit_inputs_.push_back(n);
}
dyndeps->implicit_outputs_.reserve(outs.size());
- for (vector<EvalString>::iterator i = outs.begin(); i != outs.end(); ++i) {
- string path = i->Evaluate(&env_);
+ for (vector<LexedPath>::iterator i = outs.begin(); i != outs.end(); ++i) {
+ string path;
+ EvaluatePathInScope(&path, *i, scope_);
string path_err;
uint64_t slash_bits;
if (!CanonicalizePath(&path, &slash_bits, &path_err))
- return lexer_.Error(path_err, err);
+ return lexer.Error(path_err, err);
Node* n = state_->GetNode(path, slash_bits);
dyndeps->implicit_outputs_.push_back(n);
}
diff --git a/src/dyndep_parser.h b/src/dyndep_parser.h
index 09a3722..b98541a 100644
--- a/src/dyndep_parser.h
+++ b/src/dyndep_parser.h
@@ -35,12 +35,12 @@
/// Parse a file, given its contents as a string.
bool Parse(const string& filename, const string& input, string* err);
- bool ParseDyndepVersion(string* err);
- bool ParseLet(string* key, EvalString* val, string* err);
- bool ParseEdge(string* err);
+ bool ParseDyndepVersion(Lexer& lexer, string* err);
+ bool ParseLet(Lexer& lexer, StringPiece* key, StringPiece* val, string* err);
+ bool ParseEdge(Lexer& lexer, string* err);
DyndepFile* dyndep_file_;
- BindingEnv env_;
+ ScopePosition scope_;
};
#endif // NINJA_DYNDEP_PARSER_H_
diff --git a/src/eval_env.cc b/src/eval_env.cc
index aa3d2b6..b52436b 100644
--- a/src/eval_env.cc
+++ b/src/eval_env.cc
@@ -12,57 +12,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#include <assert.h>
-
#include "eval_env.h"
-string BindingEnv::LookupVariable(const string& var) {
- map<string, string>::iterator i = bindings_.find(var);
- if (i != bindings_.end())
- return i->second;
- if (parent_)
- return parent_->LookupVariable(var);
- return "";
+#include <assert.h>
+
+#include "graph.h"
+#include "state.h"
+
+Rule::Rule(const HashedStrView& name) : name_(name) {
+ pos_.base = new BasePosition {{ &State::kBuiltinScope, 0 }}; // leaked
}
-void BindingEnv::AddBinding(const string& key, const string& val) {
- bindings_[key] = val;
-}
+bool Rule::IsReservedBinding(StringPiece var) {
+ // Cycle detection for rule variable evaluation uses a fixed recursion depth
+ // that's guaranteed to be larger than the number of reserved binding names
+ // listed below.
+ static_assert(EdgeEval::kEvalRecursionLimit == 16,
+ "Unexpected rule variable evaluation recursion limit");
-void BindingEnv::AddRule(const Rule* rule) {
- assert(LookupRuleCurrentScope(rule->name()) == NULL);
- rules_[rule->name()] = rule;
-}
-
-const Rule* BindingEnv::LookupRuleCurrentScope(const string& rule_name) {
- map<string, const Rule*>::iterator i = rules_.find(rule_name);
- if (i == rules_.end())
- return NULL;
- return i->second;
-}
-
-const Rule* BindingEnv::LookupRule(const string& rule_name) {
- map<string, const Rule*>::iterator i = rules_.find(rule_name);
- if (i != rules_.end())
- return i->second;
- if (parent_)
- return parent_->LookupRule(rule_name);
- return NULL;
-}
-
-void Rule::AddBinding(const string& key, const EvalString& val) {
- bindings_[key] = val;
-}
-
-const EvalString* Rule::GetBinding(const string& key) const {
- Bindings::const_iterator i = bindings_.find(key);
- if (i == bindings_.end())
- return NULL;
- return &i->second;
-}
-
-// static
-bool Rule::IsReservedBinding(const string& var) {
return var == "command" ||
var == "depfile" ||
var == "dyndep" ||
@@ -73,61 +40,133 @@
var == "restat" ||
var == "rspfile" ||
var == "rspfile_content" ||
+ var == "phony_output" ||
var == "msvc_deps_prefix";
}
-const map<string, const Rule*>& BindingEnv::GetRules() const {
- return rules_;
+void Binding::Evaluate(std::string* out_append) {
+ if (is_evaluated_.load()) {
+ out_append->append(final_value_);
+ return;
+ }
+
+ std::string str;
+ EvaluateBindingInScope(&str, parsed_value_, pos_.scope_pos());
+ out_append->append(str);
+
+ // Try to store the result so we can use it again later. If we can't acquire
+ // the lock, then another thread has already acquired it and will set the
+ // final binding value.
+ std::unique_lock<std::mutex> lock(mutex_, std::try_to_lock);
+ if (!lock.owns_lock())
+ return;
+
+ // Check the flag again. Another thread could have set it before this thread
+ // acquired the lock, and if it had, then other threads could already be using
+ // this binding's saved value.
+ if (is_evaluated_.load())
+ return;
+
+ final_value_ = std::move(str);
+ is_evaluated_.store(true);
}
-string BindingEnv::LookupWithFallback(const string& var,
- const EvalString* eval,
- Env* env) {
- map<string, string>::iterator i = bindings_.find(var);
- if (i != bindings_.end())
- return i->second;
+template <typename K, typename V>
+static void ExpandTable(std::unordered_map<K, V>& table, size_t extra_size) {
+ size_t needed_buckets = table.size() + extra_size + 1;
+ if (table.bucket_count() >= needed_buckets)
+ return;
+ table.rehash(needed_buckets * 2);
+};
- if (eval)
- return eval->Evaluate(env);
-
- if (parent_)
- return parent_->LookupVariable(var);
-
- return "";
+void Scope::ReserveTableSpace(size_t new_bindings, size_t new_rules) {
+ ExpandTable(bindings_, new_bindings);
+ ExpandTable(rules_, new_rules);
}
-string EvalString::Evaluate(Env* env) const {
- string result;
- for (TokenList::const_iterator i = parsed_.begin(); i != parsed_.end(); ++i) {
- if (i->second == RAW)
- result.append(i->first);
- else
- result.append(env->LookupVariable(i->first));
+std::map<std::string, const Rule*> Scope::GetRules() const {
+ std::map<std::string, const Rule*> result;
+ for (std::pair<HashedStrView, Rule*> pair : rules_) {
+ Rule* rule = pair.second;
+ result[rule->name()] = rule;
}
return result;
}
-void EvalString::AddText(StringPiece text) {
- // Add it to the end of an existing RAW token if possible.
- if (!parsed_.empty() && parsed_.back().second == RAW) {
- parsed_.back().first.append(text.str_, text.len_);
- } else {
- parsed_.push_back(make_pair(text.AsString(), RAW));
- }
-}
-void EvalString::AddSpecial(StringPiece text) {
- parsed_.push_back(make_pair(text.AsString(), SPECIAL));
+Binding* Scope::LookupBindingAtPos(const HashedStrView& var, ScopePosition pos) {
+ Scope* scope = pos.scope;
+ if (scope == nullptr) return nullptr;
+
+ auto it = scope->bindings_.find(var);
+ if (it == scope->bindings_.end())
+ return LookupBindingAtPos(var, scope->parent_);
+
+ struct BindingCmp {
+ bool operator()(Binding* x, DeclIndex y) const {
+ return x->pos_.scope_index() < y;
+ }
+ };
+
+ // A binding "Foo = $Foo" is valid; the "$Foo" is not a self-reference but
+ // a reference to the previous binding of Foo. The evaluation of "$Foo"
+ // happens with the same DeclIndex as the "Foo = $Foo" binding, so we want
+ // to find a binding whose ScopePosition is strictly less than the one we're
+ // searching for, not equal to it.
+ std::vector<Binding*>& decls = it->second;
+ auto it2 = std::lower_bound(
+ decls.begin(), decls.end(),
+ pos.index, BindingCmp());
+ if (it2 == decls.begin())
+ return LookupBindingAtPos(var, scope->parent_);
+ return *(it2 - 1);
}
-string EvalString::Serialize() const {
- string result;
- for (TokenList::const_iterator i = parsed_.begin();
- i != parsed_.end(); ++i) {
- result.append("[");
- if (i->second == SPECIAL)
- result.append("$");
- result.append(i->first);
- result.append("]");
+Binding* Scope::LookupBinding(const HashedStrView& var, Scope* scope) {
+ if (scope == nullptr) return nullptr;
+
+ auto it = scope->bindings_.find(var);
+ if (it == scope->bindings_.end()) {
+ // When we delegate to the parent scope, we match bindings at the end of the
+ // parent's scope, even if they weren't in scope when the subninja scope was
+ // parsed. The behavior matters after parsing is complete and we're
+ // evaluating edge bindings that involve rule variable expansions.
+ return LookupBinding(var, scope->parent_.scope);
}
+
+ std::vector<Binding*>& decls = it->second;
+ assert(!decls.empty());
+
+ // Evaluate the binding.
+ return decls.back();
+}
+
+void Scope::EvaluateVariableAtPos(std::string* out_append,
+ const HashedStrView& var, ScopePosition pos) {
+ if (Binding* binding = LookupBindingAtPos(var, pos))
+ binding->Evaluate(out_append);
+}
+
+void Scope::EvaluateVariable(std::string* out_append, const HashedStrView& var,
+ Scope* scope) {
+ if (Binding* binding = LookupBinding(var, scope))
+ binding->Evaluate(out_append);
+}
+
+std::string Scope::LookupVariable(const HashedStrView& var) {
+ std::string result;
+ EvaluateVariable(&result, var, this);
return result;
}
+
+Rule* Scope::LookupRuleAtPos(const HashedStrView& rule_name,
+ ScopePosition pos) {
+ Scope* scope = pos.scope;
+ if (scope == nullptr) return nullptr;
+ auto it = scope->rules_.find(rule_name);
+ if (it != scope->rules_.end()) {
+ Rule* rule = it->second;
+ if (rule->pos_.scope_index() < pos.index)
+ return rule;
+ }
+ return LookupRuleAtPos(rule_name, scope->parent_);
+}
diff --git a/src/eval_env.h b/src/eval_env.h
index 999ce42..1930226 100644
--- a/src/eval_env.h
+++ b/src/eval_env.h
@@ -15,91 +15,197 @@
#ifndef NINJA_EVAL_ENV_H_
#define NINJA_EVAL_ENV_H_
+#include <assert.h>
+#include <stdint.h>
+
#include <map>
+#include <mutex>
#include <string>
+#include <unordered_map>
#include <vector>
using namespace std;
+#include "hashed_str_view.h"
+#include "lexer.h"
#include "string_piece.h"
-struct Rule;
+using DeclIndex = uint32_t;
+const DeclIndex kLastDeclIndex = static_cast<DeclIndex>(-1);
-/// An interface for a scope for variable (e.g. "$foo") lookups.
-struct Env {
- virtual ~Env() {}
- virtual string LookupVariable(const string& var) = 0;
+struct Scope;
+
+/// Position of a declaration within a scope.
+struct ScopePosition {
+ ScopePosition(Scope* scope=nullptr, DeclIndex index=0)
+ : scope(scope), index(index) {}
+
+ Scope* scope = nullptr;
+ DeclIndex index = 0;
};
-/// A tokenized string that contains variable references.
-/// Can be evaluated relative to an Env.
-struct EvalString {
- string Evaluate(Env* env) const;
+/// Position of a parsed manifest "clump" within its containing scope and within
+/// all manifest files, in DFS order. This field is updated after parallelized
+/// parsing is complete.
+struct BasePosition {
+ BasePosition(ScopePosition scope={}, DeclIndex dfs_location=0)
+ : scope(scope), dfs_location(dfs_location) {}
- void Clear() { parsed_.clear(); }
- bool empty() const { return parsed_.empty(); }
+ /// The scope location corresponding to this chunk. (Updated during DFS
+ /// manifest parsing.)
+ ScopePosition scope = {};
- void AddText(StringPiece text);
- void AddSpecial(StringPiece text);
+ /// Location of the chunk's declarations within DFS order of all the ninja
+ /// files. This field is useful for verifying that pool references and default
+ /// target references appear *after* their referents have been declared.
+ DeclIndex dfs_location = 0;
- /// Construct a human-readable representation of the parsed state,
- /// for use in tests.
- string Serialize() const;
+ /// Check that the position is valid. (i.e. The clump it belongs to has been
+ /// assigned a scope and a DFS position.)
+ bool initialized() const {
+ return scope.scope != nullptr;
+ }
+};
+
+/// Offset of a manifest declaration relative to the beginning of a "clump" of
+/// declarations.
+struct RelativePosition {
+ RelativePosition(BasePosition* base=nullptr, DeclIndex offset=0)
+ : base(base), offset(offset) {}
+
+ BasePosition* base = nullptr;
+ DeclIndex offset = 0;
+
+ DeclIndex dfs_location() const { Validate(); return base->dfs_location + offset; }
+ DeclIndex scope_index() const { Validate(); return base->scope.index + offset; }
+ Scope* scope() const { Validate(); return base->scope.scope; }
+ ScopePosition scope_pos() const { Validate(); return { base->scope.scope, scope_index() }; }
private:
- enum TokenType { RAW, SPECIAL };
- typedef vector<pair<string, TokenType> > TokenList;
- TokenList parsed_;
+ void Validate() const {
+ assert(base->initialized() && "clump position hasn't been set yet");
+ }
};
/// An invokable build command and associated metadata (description, etc.).
struct Rule {
- explicit Rule(const string& name) : name_(name) {}
+ Rule() {}
- const string& name() const { return name_; }
+ /// This constructor is used to construct built-in rules.
+ explicit Rule(const HashedStrView& name);
- void AddBinding(const string& key, const EvalString& val);
+ const std::string& name() const { return name_.str(); }
+ const HashedStr& name_hashed() const { return name_; }
- static bool IsReservedBinding(const string& var);
+ static bool IsReservedBinding(StringPiece var);
- const EvalString* GetBinding(const string& key) const;
+ const std::string* GetBinding(const HashedStrView& name) const {
+ for (auto it = bindings_.rbegin(); it != bindings_.rend(); ++it) {
+ if (it->first == name)
+ return &it->second;
+ }
+ return nullptr;
+ }
- private:
- // Allow the parsers to reach into this object and fill out its fields.
- friend struct ManifestParser;
+ /// Temporary fields used only during manifest parsing.
+ struct {
+ /// The position of the rule in its source file. Used for diagnostics.
+ size_t rule_name_diag_pos = 0;
+ } parse_state_;
- string name_;
- typedef map<string, EvalString> Bindings;
- Bindings bindings_;
+ RelativePosition pos_;
+ HashedStr name_;
+ std::vector<std::pair<HashedStr, std::string>> bindings_;
};
-/// An Env which contains a mapping of variables to values
-/// as well as a pointer to a parent scope.
-struct BindingEnv : public Env {
- BindingEnv() : parent_(NULL) {}
- explicit BindingEnv(BindingEnv* parent) : parent_(parent) {}
+struct Binding {
+ /// This function is thread-safe. It stores a copy of the evaluated binding in
+ /// the Binding object if it hasn't been evaluated yet.
+ void Evaluate(std::string* out_append);
- virtual ~BindingEnv() {}
- virtual string LookupVariable(const string& var);
+ const std::string& name() { return name_.str(); }
+ const HashedStr& name_hashed() { return name_; }
- void AddRule(const Rule* rule);
- const Rule* LookupRule(const string& rule_name);
- const Rule* LookupRuleCurrentScope(const string& rule_name);
- const map<string, const Rule*>& GetRules() const;
+ RelativePosition pos_;
+ HashedStr name_;
- void AddBinding(const string& key, const string& val);
-
- /// This is tricky. Edges want lookup scope to go in this order:
- /// 1) value set on edge itself (edge_->env_)
- /// 2) value set on rule, with expansion in the edge's scope
- /// 3) value set on enclosing scope of edge (edge_->env_->parent_)
- /// This function takes as parameters the necessary info to do (2).
- string LookupWithFallback(const string& var, const EvalString* eval,
- Env* env);
+ /// The manifest parser initializes this field with the location of the
+ /// binding's unevaluated memory in the loaded manifest file.
+ StringPiece parsed_value_;
private:
- map<string, string> bindings_;
- map<string, const Rule*> rules_;
- BindingEnv* parent_;
+ /// This binding's value is evaluated lazily, but it must be evaluated before
+ /// the manifest files are unloaded.
+ std::atomic<bool> is_evaluated_;
+ std::mutex mutex_;
+ std::string final_value_;
+};
+
+struct Scope {
+ Scope(ScopePosition parent) : parent_(parent) {}
+
+ /// Preallocate space in the hash tables so adding bindings and rules is more
+ /// efficient.
+ void ReserveTableSpace(size_t new_bindings, size_t new_rules);
+
+ void AddBinding(Binding* binding) {
+ bindings_[binding->name_hashed()].push_back(binding);
+ }
+
+ /// Searches for a binding using a scope position (i.e. at parse time).
+ /// Returns nullptr if the binding doesn't exist.
+ static Binding* LookupBindingAtPos(const HashedStrView& var, ScopePosition pos);
+
+ /// Searches for a binding in a scope and its ancestors. The position of a
+ /// binding within its containing scope is ignored (i.e. post-parse lookup
+ /// semantics). Returns nullptr if no binding is found. The Scope pointer may
+ /// be nullptr.
+ static Binding* LookupBinding(const HashedStrView& var, Scope* scope);
+
+ /// Append a ${var} reference using a parse-time scope position.
+ static void EvaluateVariableAtPos(std::string* out_append,
+ const HashedStrView& var,
+ ScopePosition pos);
+
+ /// Append a ${var} reference using a post-parse scope, which may be nullptr.
+ static void EvaluateVariable(std::string* out_append,
+ const HashedStrView& var,
+ Scope* scope);
+
+ /// Convenience method for looking up the value of a binding after manifest
+ /// parsing is finished.
+ std::string LookupVariable(const HashedStrView& var);
+
+ bool AddRule(Rule* rule) {
+ return rules_.insert({ rule->name_hashed(), rule }).second;
+ }
+
+ static Rule* LookupRuleAtPos(const HashedStrView& rule_name,
+ ScopePosition pos);
+
+ /// Tests use this function to verify that a scope's rules are correct.
+ std::map<std::string, const Rule*> GetRules() const;
+
+ ScopePosition AllocDecls(DeclIndex count) {
+ ScopePosition result = GetCurrentEndOfScope();
+ pos_ += count;
+ return result;
+ }
+
+ ScopePosition GetCurrentEndOfScope() { return { this, pos_ }; }
+
+private:
+ /// The position of this scope within its parent scope.
+ /// (ScopePosition::parent will be nullptr for the root scope.)
+ ScopePosition parent_;
+
+ DeclIndex pos_ = 0;
+
+ std::unordered_map<HashedStrView, std::vector<Binding*>> bindings_;
+
+ /// A scope can only declare a rule with a particular name once. Even if a
+ /// scope has a rule of a given name, a lookup for that name can still find a
+ /// parent scope's rule if the search position comes before this scope's rule.
+ std::unordered_map<HashedStrView, Rule*> rules_;
};
#endif // NINJA_EVAL_ENV_H_
diff --git a/src/filebuf.h b/src/filebuf.h
new file mode 100644
index 0000000..335c594
--- /dev/null
+++ b/src/filebuf.h
@@ -0,0 +1,49 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_FILEBUF_H_
+#define NINJA_FILEBUF_H_
+
+#include <stdio.h>
+
+#include <streambuf>
+
+// A non-buffering std::streambuf implementation that allows using
+// a FILE* as an ostream.
+class ofilebuf : public std::streambuf {
+ public:
+ ofilebuf(FILE* f) : f_(f) { }
+ ~ofilebuf() { }
+
+ private:
+ int_type overflow(int_type c) {
+ if (c != traits_type::eof()) {
+ int ret = fputc(c, f_);
+ if (ret == EOF) {
+ return traits_type::eof();
+ }
+ }
+
+ return c;
+ }
+
+ std::streamsize xsputn(const char* s, std::streamsize count) {
+ return fwrite(s, 1, count, f_);
+ }
+
+ private:
+ FILE* f_;
+};
+
+#endif // NINJA_FILEBUF_H_
diff --git a/src/frontend.pb.h b/src/frontend.pb.h
new file mode 100644
index 0000000..53d2431
--- /dev/null
+++ b/src/frontend.pb.h
@@ -0,0 +1,573 @@
+// This file is autogenerated by generate_proto_header.py, do not edit
+
+#ifndef NINJA_FRONTEND_PB_H
+#define NINJA_FRONTEND_PB_H
+
+#include <inttypes.h>
+
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "proto.h"
+
+namespace ninja {
+struct Status {
+ struct TotalEdges {
+ uint32_t total_edges_;
+ bool has_total_edges_;
+
+ TotalEdges() {
+ has_total_edges_ = false;
+ total_edges_ = static_cast< uint32_t >(0);
+ }
+
+ TotalEdges(const TotalEdges&);
+ void operator=(const TotalEdges&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ WriteVarint32(output__, 1, total_edges_);
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ size += VarintSize32(total_edges_) + 1;
+ return size;
+ }
+
+ void Clear() {
+ total_edges_ = static_cast< uint32_t >(0);
+ }
+
+ uint32_t* mutable_total_edges() {
+ has_total_edges_ = true;
+ return &total_edges_;
+ }
+ void set_total_edges(const uint32_t& value) {
+ has_total_edges_ = true;
+ total_edges_ = value;
+ }
+ };
+
+ struct BuildStarted {
+ uint32_t parallelism_;
+ bool has_parallelism_;
+ bool verbose_;
+ bool has_verbose_;
+
+ BuildStarted() {
+ has_parallelism_ = false;
+ parallelism_ = static_cast< uint32_t >(0);
+ has_verbose_ = false;
+ verbose_ = static_cast< bool >(0);
+ }
+
+ BuildStarted(const BuildStarted&);
+ void operator=(const BuildStarted&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ WriteVarint32(output__, 1, parallelism_);
+ WriteVarint32(output__, 2, verbose_);
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ size += VarintSize32(parallelism_) + 1;
+ size += VarintSizeBool(verbose_) + 1;
+ return size;
+ }
+
+ void Clear() {
+ parallelism_ = static_cast< uint32_t >(0);
+ verbose_ = static_cast< bool >(0);
+ }
+
+ uint32_t* mutable_parallelism() {
+ has_parallelism_ = true;
+ return ¶llelism_;
+ }
+ void set_parallelism(const uint32_t& value) {
+ has_parallelism_ = true;
+ parallelism_ = value;
+ }
+ bool* mutable_verbose() {
+ has_verbose_ = true;
+ return &verbose_;
+ }
+ void set_verbose(const bool& value) {
+ has_verbose_ = true;
+ verbose_ = value;
+ }
+ };
+
+ struct BuildFinished {
+ BuildFinished() {
+ }
+
+ BuildFinished(const BuildFinished&);
+ void operator=(const BuildFinished&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ return size;
+ }
+
+ void Clear() {
+ }
+
+ };
+
+ struct EdgeStarted {
+ uint32_t id_;
+ bool has_id_;
+ uint32_t start_time_;
+ bool has_start_time_;
+ std::vector< std::string > inputs_;
+ bool has_inputs_;
+ std::vector< std::string > outputs_;
+ bool has_outputs_;
+ std::string desc_;
+ bool has_desc_;
+ std::string command_;
+ bool has_command_;
+ bool console_;
+ bool has_console_;
+
+ EdgeStarted() {
+ has_id_ = false;
+ id_ = static_cast< uint32_t >(0);
+ has_start_time_ = false;
+ start_time_ = static_cast< uint32_t >(0);
+ has_inputs_ = false;
+ has_outputs_ = false;
+ has_desc_ = false;
+ has_command_ = false;
+ has_console_ = false;
+ console_ = static_cast< bool >(0);
+ }
+
+ EdgeStarted(const EdgeStarted&);
+ void operator=(const EdgeStarted&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ WriteVarint32(output__, 1, id_);
+ WriteVarint32(output__, 2, start_time_);
+ for (std::vector< std::string >::const_iterator it_ = inputs_.begin();
+ it_ != inputs_.end(); it_++) {
+ WriteString(output__, 3, *it_);
+ }
+ for (std::vector< std::string >::const_iterator it_ = outputs_.begin();
+ it_ != outputs_.end(); it_++) {
+ WriteString(output__, 4, *it_);
+ }
+ WriteString(output__, 5, desc_);
+ WriteString(output__, 6, command_);
+ WriteVarint32(output__, 7, console_);
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ size += VarintSize32(id_) + 1;
+ size += VarintSize32(start_time_) + 1;
+ for (std::vector< std::string >::const_iterator it_ = inputs_.begin();
+ it_ != inputs_.end(); it_++) {
+ size += StringSize(*it_) + 1;
+ }
+ for (std::vector< std::string >::const_iterator it_ = outputs_.begin();
+ it_ != outputs_.end(); it_++) {
+ size += StringSize(*it_) + 1;
+ }
+ size += StringSize(desc_) + 1;
+ size += StringSize(command_) + 1;
+ size += VarintSizeBool(console_) + 1;
+ return size;
+ }
+
+ void Clear() {
+ id_ = static_cast< uint32_t >(0);
+ start_time_ = static_cast< uint32_t >(0);
+ inputs_.clear();
+ outputs_.clear();
+ desc_.clear();
+ command_.clear();
+ console_ = static_cast< bool >(0);
+ }
+
+ uint32_t* mutable_id() {
+ has_id_ = true;
+ return &id_;
+ }
+ void set_id(const uint32_t& value) {
+ has_id_ = true;
+ id_ = value;
+ }
+ uint32_t* mutable_start_time() {
+ has_start_time_ = true;
+ return &start_time_;
+ }
+ void set_start_time(const uint32_t& value) {
+ has_start_time_ = true;
+ start_time_ = value;
+ }
+ std::vector< std::string >* mutable_inputs() {
+ has_inputs_ = true;
+ return &inputs_;
+ }
+ void add_inputs(const std::string& value) {
+ has_inputs_ = true;
+ inputs_.push_back(value);
+ }
+ void set_inputs(const std::vector< std::string >& value) {
+ has_inputs_ = true;
+ inputs_ = value;
+ }
+ std::vector< std::string >* mutable_outputs() {
+ has_outputs_ = true;
+ return &outputs_;
+ }
+ void add_outputs(const std::string& value) {
+ has_outputs_ = true;
+ outputs_.push_back(value);
+ }
+ void set_outputs(const std::vector< std::string >& value) {
+ has_outputs_ = true;
+ outputs_ = value;
+ }
+ std::string* mutable_desc() {
+ has_desc_ = true;
+ return &desc_;
+ }
+ void set_desc(const std::string& value) {
+ has_desc_ = true;
+ desc_ = value;
+ }
+ std::string* mutable_command() {
+ has_command_ = true;
+ return &command_;
+ }
+ void set_command(const std::string& value) {
+ has_command_ = true;
+ command_ = value;
+ }
+ bool* mutable_console() {
+ has_console_ = true;
+ return &console_;
+ }
+ void set_console(const bool& value) {
+ has_console_ = true;
+ console_ = value;
+ }
+ };
+
+ struct EdgeFinished {
+ uint32_t id_;
+ bool has_id_;
+ uint32_t end_time_;
+ bool has_end_time_;
+ int32_t status_;
+ bool has_status_;
+ std::string output_;
+ bool has_output_;
+ uint32_t user_time_;
+ bool has_user_time_;
+ uint32_t system_time_;
+ bool has_system_time_;
+
+ EdgeFinished() {
+ has_id_ = false;
+ id_ = static_cast< uint32_t >(0);
+ has_end_time_ = false;
+ end_time_ = static_cast< uint32_t >(0);
+ has_status_ = false;
+ status_ = static_cast< int32_t >(0);
+ has_output_ = false;
+ has_user_time_ = false;
+ user_time_ = static_cast< uint32_t >(0);
+ has_system_time_ = false;
+ system_time_ = static_cast< uint32_t >(0);
+ }
+
+ EdgeFinished(const EdgeFinished&);
+ void operator=(const EdgeFinished&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ WriteVarint32(output__, 1, id_);
+ WriteVarint32(output__, 2, end_time_);
+ WriteVarint32(output__, 3, ZigZagEncode32(status_));
+ WriteString(output__, 4, output_);
+ WriteVarint32(output__, 5, user_time_);
+ WriteVarint32(output__, 6, system_time_);
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ size += VarintSize32(id_) + 1;
+ size += VarintSize32(end_time_) + 1;
+ size += VarintSize32(ZigZagEncode32(status_)) + 1;
+ size += StringSize(output_) + 1;
+ size += VarintSize32(user_time_) + 1;
+ size += VarintSize32(system_time_) + 1;
+ return size;
+ }
+
+ void Clear() {
+ id_ = static_cast< uint32_t >(0);
+ end_time_ = static_cast< uint32_t >(0);
+ status_ = static_cast< int32_t >(0);
+ output_.clear();
+ user_time_ = static_cast< uint32_t >(0);
+ system_time_ = static_cast< uint32_t >(0);
+ }
+
+ uint32_t* mutable_id() {
+ has_id_ = true;
+ return &id_;
+ }
+ void set_id(const uint32_t& value) {
+ has_id_ = true;
+ id_ = value;
+ }
+ uint32_t* mutable_end_time() {
+ has_end_time_ = true;
+ return &end_time_;
+ }
+ void set_end_time(const uint32_t& value) {
+ has_end_time_ = true;
+ end_time_ = value;
+ }
+ int32_t* mutable_status() {
+ has_status_ = true;
+ return &status_;
+ }
+ void set_status(const int32_t& value) {
+ has_status_ = true;
+ status_ = value;
+ }
+ std::string* mutable_output() {
+ has_output_ = true;
+ return &output_;
+ }
+ void set_output(const std::string& value) {
+ has_output_ = true;
+ output_ = value;
+ }
+ uint32_t* mutable_user_time() {
+ has_user_time_ = true;
+ return &user_time_;
+ }
+ void set_user_time(const uint32_t& value) {
+ has_user_time_ = true;
+ user_time_ = value;
+ }
+ uint32_t* mutable_system_time() {
+ has_system_time_ = true;
+ return &system_time_;
+ }
+ void set_system_time(const uint32_t& value) {
+ has_system_time_ = true;
+ system_time_ = value;
+ }
+ };
+
+ struct Message {
+ enum Level {
+ INFO = 0,
+ WARNING = 1,
+ ERROR = 2,
+ DEBUG = 3,
+ };
+
+ ::ninja::Status::Message::Level level_;
+ bool has_level_;
+ std::string message_;
+ bool has_message_;
+
+ Message() {
+ has_level_ = false;
+ level_ = static_cast< ::ninja::Status::Message::Level >(0);
+ has_message_ = false;
+ }
+
+ Message(const Message&);
+ void operator=(const Message&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ WriteVarint32SignExtended(output__, 1, static_cast<int32_t>(level_));
+ WriteString(output__, 2, message_);
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ size += VarintSize32SignExtended(static_cast<int32_t>(level_)) + 1;
+ size += StringSize(message_) + 1;
+ return size;
+ }
+
+ void Clear() {
+ level_ = static_cast< ::ninja::Status::Message::Level >(0);
+ message_.clear();
+ }
+
+ ::ninja::Status::Message::Level* mutable_level() {
+ has_level_ = true;
+ return &level_;
+ }
+ void set_level(const ::ninja::Status::Message::Level& value) {
+ has_level_ = true;
+ level_ = value;
+ }
+ std::string* mutable_message() {
+ has_message_ = true;
+ return &message_;
+ }
+ void set_message(const std::string& value) {
+ has_message_ = true;
+ message_ = value;
+ }
+ };
+
+ ::ninja::Status::TotalEdges total_edges_;
+ bool has_total_edges_;
+ ::ninja::Status::BuildStarted build_started_;
+ bool has_build_started_;
+ ::ninja::Status::BuildFinished build_finished_;
+ bool has_build_finished_;
+ ::ninja::Status::EdgeStarted edge_started_;
+ bool has_edge_started_;
+ ::ninja::Status::EdgeFinished edge_finished_;
+ bool has_edge_finished_;
+ ::ninja::Status::Message message_;
+ bool has_message_;
+
+ Status() {
+ has_total_edges_ = false;
+ has_build_started_ = false;
+ has_build_finished_ = false;
+ has_edge_started_ = false;
+ has_edge_finished_ = false;
+ has_message_ = false;
+ }
+
+ Status(const Status&);
+ void operator=(const Status&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ if (has_total_edges_) {
+ WriteLengthDelimited(output__, 1,
+ total_edges_.ByteSizeLong());
+ total_edges_.SerializeToOstream(output__);
+ }
+ if (has_build_started_) {
+ WriteLengthDelimited(output__, 2,
+ build_started_.ByteSizeLong());
+ build_started_.SerializeToOstream(output__);
+ }
+ if (has_build_finished_) {
+ WriteLengthDelimited(output__, 3,
+ build_finished_.ByteSizeLong());
+ build_finished_.SerializeToOstream(output__);
+ }
+ if (has_edge_started_) {
+ WriteLengthDelimited(output__, 4,
+ edge_started_.ByteSizeLong());
+ edge_started_.SerializeToOstream(output__);
+ }
+ if (has_edge_finished_) {
+ WriteLengthDelimited(output__, 5,
+ edge_finished_.ByteSizeLong());
+ edge_finished_.SerializeToOstream(output__);
+ }
+ if (has_message_) {
+ WriteLengthDelimited(output__, 6,
+ message_.ByteSizeLong());
+ message_.SerializeToOstream(output__);
+ }
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ if (has_total_edges_) {
+ size += 1 + VarintSize32(total_edges_.ByteSizeLong());
+ size += total_edges_.ByteSizeLong();
+ }
+ if (has_build_started_) {
+ size += 1 + VarintSize32(build_started_.ByteSizeLong());
+ size += build_started_.ByteSizeLong();
+ }
+ if (has_build_finished_) {
+ size += 1 + VarintSize32(build_finished_.ByteSizeLong());
+ size += build_finished_.ByteSizeLong();
+ }
+ if (has_edge_started_) {
+ size += 1 + VarintSize32(edge_started_.ByteSizeLong());
+ size += edge_started_.ByteSizeLong();
+ }
+ if (has_edge_finished_) {
+ size += 1 + VarintSize32(edge_finished_.ByteSizeLong());
+ size += edge_finished_.ByteSizeLong();
+ }
+ if (has_message_) {
+ size += 1 + VarintSize32(message_.ByteSizeLong());
+ size += message_.ByteSizeLong();
+ }
+ return size;
+ }
+
+ void Clear() {
+ if (has_total_edges_) {
+ total_edges_.Clear();
+ has_total_edges_ = false;
+ }
+ if (has_build_started_) {
+ build_started_.Clear();
+ has_build_started_ = false;
+ }
+ if (has_build_finished_) {
+ build_finished_.Clear();
+ has_build_finished_ = false;
+ }
+ if (has_edge_started_) {
+ edge_started_.Clear();
+ has_edge_started_ = false;
+ }
+ if (has_edge_finished_) {
+ edge_finished_.Clear();
+ has_edge_finished_ = false;
+ }
+ if (has_message_) {
+ message_.Clear();
+ has_message_ = false;
+ }
+ }
+
+ ::ninja::Status::TotalEdges* mutable_total_edges() {
+ has_total_edges_ = true;
+ return &total_edges_;
+ }
+ ::ninja::Status::BuildStarted* mutable_build_started() {
+ has_build_started_ = true;
+ return &build_started_;
+ }
+ ::ninja::Status::BuildFinished* mutable_build_finished() {
+ has_build_finished_ = true;
+ return &build_finished_;
+ }
+ ::ninja::Status::EdgeStarted* mutable_edge_started() {
+ has_edge_started_ = true;
+ return &edge_started_;
+ }
+ ::ninja::Status::EdgeFinished* mutable_edge_finished() {
+ has_edge_finished_ = true;
+ return &edge_finished_;
+ }
+ ::ninja::Status::Message* mutable_message() {
+ has_message_ = true;
+ return &message_;
+ }
+};
+
+}
+#endif // NINJA_FRONTEND_PB_H
diff --git a/src/frontend.proto b/src/frontend.proto
new file mode 100644
index 0000000..57423d8
--- /dev/null
+++ b/src/frontend.proto
@@ -0,0 +1,88 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+option optimize_for = LITE_RUNTIME;
+
+package ninja;
+
+message Status {
+ message TotalEdges {
+ // New value for total edges in the build.
+ optional uint32 total_edges = 1;
+ }
+
+ message BuildStarted {
+ // Number of jobs Ninja will run in parallel.
+ optional uint32 parallelism = 1;
+ // Verbose value passed to ninja.
+ optional bool verbose = 2;
+ }
+
+ message BuildFinished {
+ }
+
+ message EdgeStarted {
+ // Edge identification number, unique to a Ninja run.
+ optional uint32 id = 1;
+ // Edge start time in milliseconds since Ninja started.
+ optional uint32 start_time = 2;
+ // List of edge inputs.
+ repeated string inputs = 3;
+ // List of edge outputs.
+ repeated string outputs = 4;
+ // Description field from the edge.
+ optional string desc = 5;
+ // Command field from the edge.
+ optional string command = 6;
+ // Edge uses console.
+ optional bool console = 7;
+ }
+
+ message EdgeFinished {
+ // Edge identification number, unique to a Ninja run.
+ optional uint32 id = 1;
+ // Edge end time in milliseconds since Ninja started.
+ optional uint32 end_time = 2;
+ // Exit status (0 for success).
+ optional sint32 status = 3;
+ // Edge output, may contain ANSI codes.
+ optional string output = 4;
+ // Number of milliseconds spent executing in user mode
+ optional uint32 user_time = 5;
+ // Number of milliseconds spent executing in kernel mode
+ optional uint32 system_time = 6;
+ }
+
+ message Message {
+ enum Level {
+ INFO = 0;
+ WARNING = 1;
+ ERROR = 2;
+ DEBUG = 3;
+ }
+ // Message priority level (DEBUG, INFO, WARNING, ERROR).
+ optional Level level = 1 [default = INFO];
+ // Info/warning/error message from Ninja.
+ optional string message = 2;
+ }
+
+ optional TotalEdges total_edges = 1;
+ optional BuildStarted build_started = 2;
+ optional BuildFinished build_finished = 3;
+ optional EdgeStarted edge_started = 4;
+ optional EdgeFinished edge_finished = 5;
+ optional Message message = 6;
+}
diff --git a/src/graph.cc b/src/graph.cc
index add7868..da0e44a 100644
--- a/src/graph.cc
+++ b/src/graph.cc
@@ -14,6 +14,7 @@
#include "graph.h"
+#include <deque>
#include <assert.h>
#include <stdio.h>
@@ -22,22 +23,187 @@
#include "depfile_parser.h"
#include "deps_log.h"
#include "disk_interface.h"
-#include "manifest_parser.h"
#include "metrics.h"
+#include "parallel_map.h"
#include "state.h"
#include "util.h"
+bool Node::PrecomputeStat(DiskInterface* disk_interface, std::string* err) {
+ if (in_edge() != nullptr) {
+ if (in_edge()->IsPhonyOutput()) {
+ return true;
+ }
+ return (precomputed_mtime_ = disk_interface->LStat(path_.str(), nullptr, err)) != -1;
+ } else {
+ return (precomputed_mtime_ = disk_interface->Stat(path_.str(), err)) != -1;
+ }
+}
+
bool Node::Stat(DiskInterface* disk_interface, string* err) {
- return (mtime_ = disk_interface->Stat(path_, err)) != -1;
+ if (in_edge() != nullptr) {
+ assert(!in_edge()->IsPhonyOutput());
+ return (mtime_ = disk_interface->LStat(path_.str(), nullptr, err)) != -1;
+ } else {
+ return (mtime_ = disk_interface->Stat(path_.str(), err)) != -1;
+ }
}
-bool DependencyScan::RecomputeDirty(Node* node, string* err) {
- vector<Node*> stack;
- return RecomputeDirty(node, &stack, err);
+bool Node::LStat(DiskInterface* disk_interface, bool* is_dir, string* err) {
+ assert(in_edge() != nullptr);
+ assert(!in_edge()->IsPhonyOutput());
+ return (mtime_ = disk_interface->LStat(path_.str(), is_dir, err)) != -1;
}
-bool DependencyScan::RecomputeDirty(Node* node, vector<Node*>* stack,
- string* err) {
+bool DependencyScan::RecomputeNodesDirty(const std::vector<Node*>& initial_nodes,
+ std::vector<Node*>* validation_nodes,
+ std::string* err) {
+ METRIC_RECORD("dep scan");
+ std::vector<Node*> all_nodes;
+ std::vector<Edge*> all_edges;
+ std::unique_ptr<ThreadPool> thread_pool = CreateThreadPool();
+
+ {
+ METRIC_RECORD("dep scan : collect nodes+edges");
+ for (Node* node : initial_nodes)
+ CollectPrecomputeLists(node, &all_nodes, &all_edges);
+ }
+
+ bool success = true;
+ if (!PrecomputeNodesDirty(all_nodes, all_edges, thread_pool.get(), err))
+ success = false;
+
+
+ std::deque<Node*> nodes(initial_nodes.begin(), initial_nodes.end());
+
+ if (success) {
+ METRIC_RECORD("dep scan : main pass");
+ std::vector<Node*> stack;
+ std::vector<Node*> new_validation_nodes;
+ while (!nodes.empty()) {
+ Node* node = nodes.front();
+ nodes.pop_front();
+
+ stack.clear();
+ new_validation_nodes.clear();
+ if (!RecomputeNodeDirty(node, &stack, &new_validation_nodes, err)) {
+ success = false;
+ break;
+ }
+ nodes.insert(nodes.end(), new_validation_nodes.begin(),
+ new_validation_nodes.end());
+ if (!new_validation_nodes.empty()) {
+ assert(validation_nodes &&
+ "validations require RecomputeDirty to be called with validation_nodes");
+ validation_nodes->insert(validation_nodes->end(),
+ new_validation_nodes.begin(),
+ new_validation_nodes.end());
+ }
+ }
+ }
+
+ {
+ // Ensure that the precomputed mtime information can't be used after this
+ // dependency scan finishes.
+ METRIC_RECORD("dep scan : clear pre-stat");
+ ParallelMap(thread_pool.get(), all_nodes, [](Node* node) {
+ node->ClearPrecomputedStat();
+ });
+ }
+
+ return success;
+}
+
+void DependencyScan::CollectPrecomputeLists(Node* node,
+ std::vector<Node*>* nodes,
+ std::vector<Edge*>* edges) {
+ if (node->precomputed_dirtiness())
+ return;
+ node->set_precomputed_dirtiness(true);
+ nodes->push_back(node);
+
+ Edge* edge = node->in_edge();
+ if (edge && !edge->precomputed_dirtiness_) {
+ edge->precomputed_dirtiness_ = true;
+ edges->push_back(edge);
+
+ for (Node* node : edge->inputs_) {
+ // Duplicate the dirtiness check here to avoid an unnecessary function
+ // call. (The precomputed_dirtiness() will be inlined, but the recursive
+ // call can't be.)
+ if (!node->precomputed_dirtiness())
+ CollectPrecomputeLists(node, nodes, edges);
+ }
+
+ if (!edge->validations_.empty()) {
+ for (Node* node : edge->validations_) {
+ // Duplicate the dirtiness check here to avoid an unnecessary function
+ // call. (The precomputed_dirtiness() will be inlined, but the recursive
+ // call can't be.)
+ if (!node->precomputed_dirtiness())
+ CollectPrecomputeLists(node, nodes, edges);
+ }
+ }
+ }
+
+ // Collect dependencies from the deps log. This pass could also examine
+ // depfiles, but it would be a more intrusive design change, because we don't
+ // want to parse a depfile twice.
+ if (DepsLog::Deps* deps = deps_log()->GetDeps(node)) {
+ for (int i = 0; i < deps->node_count; ++i) {
+ Node* node = deps->nodes[i];
+ // Duplicate the dirtiness check here to avoid an unnecessary function
+ // call.
+ if (!node->precomputed_dirtiness()) {
+ CollectPrecomputeLists(node, nodes, edges);
+ }
+ }
+ }
+}
+
+bool DependencyScan::PrecomputeNodesDirty(const std::vector<Node*>& nodes,
+ const std::vector<Edge*>& edges,
+ ThreadPool* thread_pool,
+ std::string* err) {
+ // Optimize the "null build" case by calling Stat in parallel on every node in
+ // the transitive closure.
+ //
+ // The Windows RealDiskInterface::Stat uses a directory-based cache that isn't
+ // thread-safe. Various tests also uses a non-thread-safe Stat, so disable the
+ // parallelized stat'ing for them as well.
+ if (disk_interface_->IsStatThreadSafe() &&
+ GetOptimalThreadPoolJobCount() > 1) {
+ METRIC_RECORD("dep scan : pre-stat nodes");
+ if (!PropagateError(err, ParallelMap(thread_pool, nodes,
+ [this](Node* node) {
+ // Each node is guaranteed to appear at most once in the collected list
+ // of nodes, so it's safe to modify the nodes from worker threads.
+ std::string err;
+ node->PrecomputeStat(disk_interface_, &err);
+ return err;
+ }))) {
+ return false;
+ }
+ }
+
+ {
+ METRIC_RECORD("dep scan : precompute edge info");
+ if (!PropagateError(err, ParallelMap(thread_pool, edges, [](Edge* edge) {
+ // As with the node list, each edge appears at most once in the
+ // collected list, so it's safe to modify the edges from worker threads.
+ std::string err;
+ edge->PrecomputeDepScanInfo(&err);
+ return err;
+ }))) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool DependencyScan::RecomputeNodeDirty(Node* node, std::vector<Node*>* stack,
+ std::vector<Node*>* validation_nodes,
+ std::string* err) {
Edge* edge = node->in_edge();
if (!edge) {
// If we already visited this leaf node then we are done.
@@ -65,68 +231,95 @@
stack->push_back(node);
bool dirty = false;
+ bool phony_output = edge->IsPhonyOutput();
edge->outputs_ready_ = true;
edge->deps_missing_ = false;
- if (!edge->deps_loaded_) {
- // This is our first encounter with this edge.
- // If there is a pending dyndep file, visit it now:
- // * If the dyndep file is ready then load it now to get any
- // additional inputs and outputs for this and other edges.
- // Once the dyndep file is loaded it will no longer be pending
- // if any other edges encounter it, but they will already have
- // been updated.
- // * If the dyndep file is not ready then since is known to be an
- // input to this edge, the edge will not be considered ready below.
- // Later during the build the dyndep file will become ready and be
- // loaded to update this edge before it can possibly be scheduled.
- if (edge->dyndep_ && edge->dyndep_->dyndep_pending()) {
- if (!RecomputeDirty(edge->dyndep_, stack, err))
- return false;
+ if (phony_output) {
+ EXPLAIN("edge with output %s is a phony output, so is always dirty",
+ node->path().c_str());
+ dirty = true;
- if (!edge->dyndep_->in_edge() ||
- edge->dyndep_->in_edge()->outputs_ready()) {
- // The dyndep file is ready, so load it now.
- if (!LoadDyndeps(edge->dyndep_, err))
+ if (edge->UsesDepsLog() || edge->UsesDepfile()) {
+ *err = "phony output " + node->path() + " has deps, which does not make sense.";
+ return false;
+ }
+ } else {
+ if (!edge->deps_loaded_) {
+ // This is our first encounter with this edge.
+ // If there is a pending dyndep file, visit it now:
+ // * If the dyndep file is ready then load it now to get any
+ // additional inputs and outputs for this and other edges.
+ // Once the dyndep file is loaded it will no longer be pending
+ // if any other edges encounter it, but they will already have
+ // been updated.
+ // * If the dyndep file is not ready then since is known to be an
+ // input to this edge, the edge will not be considered ready below.
+ // Later during the build the dyndep file will become ready and be
+ // loaded to update this edge before it can possibly be scheduled.
+ if (edge->dyndep_ && edge->dyndep_->dyndep_pending()) {
+ if (!RecomputeDirty(edge->dyndep_, stack, err))
return false;
+
+ if (!edge->dyndep_->in_edge() ||
+ edge->dyndep_->in_edge()->outputs_ready()) {
+ // The dyndep file is ready, so load it now.
+ if (!LoadDyndeps(edge->dyndep_, err))
+ return false;
+ }
+ }
+ }
+
+ // Load output mtimes so we can compare them to the most recent input below.
+ for (vector<Node*>::iterator o = edge->outputs_.begin();
+ o != edge->outputs_.end(); ++o) {
+ if (!(*o)->StatIfNecessary(disk_interface_, err))
+ return false;
+ }
+
+ if (!edge->deps_loaded_) {
+ // This is our first encounter with this edge. Load discovered deps.
+ edge->deps_loaded_ = true;
+ if (!dep_loader_.LoadDeps(edge, err)) {
+ if (!err->empty())
+ return false;
+ // Failed to load dependency info: rebuild to regenerate it.
+ // LoadDeps() did EXPLAIN() already, no need to do it here.
+ dirty = edge->deps_missing_ = true;
}
}
}
- // Load output mtimes so we can compare them to the most recent input below.
- for (vector<Node*>::iterator o = edge->outputs_.begin();
- o != edge->outputs_.end(); ++o) {
- if (!(*o)->StatIfNecessary(disk_interface_, err))
- return false;
- }
-
- if (!edge->deps_loaded_) {
- // This is our first encounter with this edge. Load discovered deps.
- edge->deps_loaded_ = true;
- if (!dep_loader_.LoadDeps(edge, err)) {
- if (!err->empty())
- return false;
- // Failed to load dependency info: rebuild to regenerate it.
- // LoadDeps() did EXPLAIN() already, no need to do it here.
- dirty = edge->deps_missing_ = true;
- }
- }
+ // Store any validation nodes from the edge for adding to the initial
+ // nodes. Don't recurse into them, that would trigger the dependency
+ // cycle detector if the validation node depends on this node.
+ // RecomputeNodesDirty will add the validation nodes to the initial nodes
+ // and recurse into them.
+ validation_nodes->insert(validation_nodes->end(),
+ edge->validations_.begin(), edge->validations_.end());
// Visit all inputs; we're dirty if any of the inputs are dirty.
Node* most_recent_input = NULL;
for (vector<Node*>::iterator i = edge->inputs_.begin();
i != edge->inputs_.end(); ++i) {
// Visit this input.
- if (!RecomputeDirty(*i, stack, err))
+ if (!RecomputeNodeDirty(*i, stack, validation_nodes, err))
return false;
// If an input is not ready, neither are our outputs.
- if (Edge* in_edge = (*i)->in_edge()) {
+ Edge* in_edge = (*i)->in_edge();
+ if (in_edge != nullptr) {
if (!in_edge->outputs_ready_)
edge->outputs_ready_ = false;
}
- if (!edge->is_order_only(i - edge->inputs_.begin())) {
+ if (!phony_output && !edge->is_order_only(i - edge->inputs_.begin())) {
+ if (in_edge != nullptr && in_edge->IsPhonyOutput()) {
+ *err = "real file '" + node->path() +
+ "' depends on phony output '" + (*i)->path() + "'\n";
+ return false;
+ }
+
// If a regular input is dirty (or missing), we're dirty.
// Otherwise consider mtime.
if ((*i)->dirty()) {
@@ -211,10 +404,30 @@
bool DependencyScan::RecomputeOutputsDirty(Edge* edge, Node* most_recent_input,
bool* outputs_dirty, string* err) {
- string command = edge->EvaluateCommand(/*incl_rsp_file=*/true);
+ assert(!edge->IsPhonyOutput());
+
+ uint64_t command_hash = edge->GetCommandHash();
for (vector<Node*>::iterator o = edge->outputs_.begin();
o != edge->outputs_.end(); ++o) {
- if (RecomputeOutputDirty(edge, most_recent_input, command, *o)) {
+ if (edge->is_phony()) {
+ // Phony edges don't write any output. Outputs are only dirty if
+ // there are no inputs and we're missing the output.
+ if (edge->inputs_.empty() && !(*o)->exists()) {
+ // For phony targets defined in the ninja file, error when using dirty phony edges.
+ // The phony edges automatically created from depfiles still need the old behavior.
+ if (missing_phony_is_err_ && !edge->phony_from_depfile_) {
+ *err = "output " + (*o)->path() + " of phony edge doesn't exist. Missing 'phony_output = true'?";
+ return false;
+ } else {
+ EXPLAIN("output %s of phony edge with no inputs doesn't exist",
+ (*o)->path().c_str());
+ *outputs_dirty = true;
+ return true;
+ }
+ }
+ continue;
+ }
+ if (RecomputeOutputDirty(edge, most_recent_input, command_hash, *o)) {
*outputs_dirty = true;
return true;
}
@@ -224,18 +437,9 @@
bool DependencyScan::RecomputeOutputDirty(Edge* edge,
Node* most_recent_input,
- const string& command,
+ uint64_t command_hash,
Node* output) {
- if (edge->is_phony()) {
- // Phony edges don't write any output. Outputs are only dirty if
- // there are no inputs and we're missing the output.
- if (edge->inputs_.empty() && !output->exists()) {
- EXPLAIN("output %s of phony edge with no inputs doesn't exist",
- output->path().c_str());
- return true;
- }
- return false;
- }
+ assert(!edge->is_phony());
BuildLog::LogEntry* entry = 0;
@@ -254,8 +458,8 @@
// build log. Use that mtime instead, so that the file will only be
// considered dirty if an input was modified since the previous run.
bool used_restat = false;
- if (edge->GetBindingBool("restat") && build_log() &&
- (entry = build_log()->LookupByOutput(output->path()))) {
+ if (edge->IsRestat() && build_log() &&
+ (entry = build_log()->LookupByOutput(output->path_hashed()))) {
output_mtime = entry->mtime;
used_restat = true;
}
@@ -271,10 +475,10 @@
}
if (build_log()) {
- bool generator = edge->GetBindingBool("generator");
- if (entry || (entry = build_log()->LookupByOutput(output->path()))) {
+ bool generator = edge->IsGenerator();
+ if (entry || (entry = build_log()->LookupByOutput(output->path_hashed()))) {
if (!generator &&
- BuildLog::LogEntry::HashCommand(command) != entry->command_hash) {
+ command_hash != entry->command_hash) {
// May also be dirty due to the command changing since the last build.
// But if this is a generator rule, the command changing does not make us
// dirty.
@@ -319,116 +523,192 @@
return true;
}
-/// An Env for an Edge, providing $in and $out.
-struct EdgeEnv : public Env {
- enum EscapeKind { kShellEscape, kDoNotEscape };
+static const HashedStrView kIn { "in" };
+static const HashedStrView kInNewline { "in_newline" };
+static const HashedStrView kOut { "out" };
- EdgeEnv(Edge* edge, EscapeKind escape)
- : edge_(edge), escape_in_out_(escape), recursive_(false) {}
- virtual string LookupVariable(const string& var);
-
- /// Given a span of Nodes, construct a list of paths suitable for a command
- /// line.
- string MakePathList(vector<Node*>::iterator begin,
- vector<Node*>::iterator end,
- char sep);
-
- private:
- vector<string> lookups_;
- Edge* edge_;
- EscapeKind escape_in_out_;
- bool recursive_;
-};
-
-string EdgeEnv::LookupVariable(const string& var) {
- if (var == "in" || var == "in_newline") {
+bool EdgeEval::EvaluateVariable(std::string* out_append,
+ const HashedStrView& var,
+ std::string* err) {
+ if (var == kIn || var == kInNewline) {
int explicit_deps_count = edge_->inputs_.size() - edge_->implicit_deps_ -
edge_->order_only_deps_;
- return MakePathList(edge_->inputs_.begin(),
- edge_->inputs_.begin() + explicit_deps_count,
- var == "in" ? ' ' : '\n');
- } else if (var == "out") {
+ AppendPathList(out_append,
+ edge_->inputs_.begin(),
+ edge_->inputs_.begin() + explicit_deps_count,
+ var == kIn ? ' ' : '\n');
+ return true;
+ } else if (var == kOut) {
int explicit_outs_count = edge_->outputs_.size() - edge_->implicit_outs_;
- return MakePathList(edge_->outputs_.begin(),
- edge_->outputs_.begin() + explicit_outs_count,
- ' ');
+ AppendPathList(out_append,
+ edge_->outputs_.begin(),
+ edge_->outputs_.begin() + explicit_outs_count,
+ ' ');
+ return true;
}
- if (recursive_) {
- vector<string>::const_iterator it;
- if ((it = find(lookups_.begin(), lookups_.end(), var)) != lookups_.end()) {
- string cycle;
- for (; it != lookups_.end(); ++it)
- cycle.append(*it + " -> ");
- cycle.append(var);
- Fatal(("cycle in rule variables: " + cycle).c_str());
+ if (edge_->EvaluateVariableSelfOnly(out_append, var))
+ return true;
+
+ // Search for a matching rule binding.
+ if (const std::string* binding_pattern = edge_->rule().GetBinding(var)) {
+ // Detect recursive rule variable usage.
+ if (recursion_count_ == kEvalRecursionLimit) {
+ std::string cycle = recursion_vars_[0].AsString();
+ for (int i = 1; i < kEvalRecursionLimit; ++i) {
+ cycle += " -> " + recursion_vars_[i].AsString();
+ if (recursion_vars_[i] == recursion_vars_[0])
+ break;
+ }
+ *err = "cycle in rule variables: " + cycle;
+ return false;
}
+ recursion_vars_[recursion_count_++] = var.str_view();
+
+ return EvaluateBindingOnRule(out_append, *binding_pattern, this, err);
}
- // See notes on BindingEnv::LookupWithFallback.
- const EvalString* eval = edge_->rule_->GetBinding(var);
- if (recursive_ && eval)
- lookups_.push_back(var);
-
- // In practice, variables defined on rules never use another rule variable.
- // For performance, only start checking for cycles after the first lookup.
- recursive_ = true;
- return edge_->env_->LookupWithFallback(var, eval, this);
+ // Fall back to the edge's enclosing scope.
+ if (eval_phase_ == EdgeEval::kParseTime) {
+ Scope::EvaluateVariableAtPos(out_append, var, edge_->pos_.scope_pos());
+ } else {
+ Scope::EvaluateVariable(out_append, var, edge_->pos_.scope());
+ }
+ return true;
}
-string EdgeEnv::MakePathList(vector<Node*>::iterator begin,
- vector<Node*>::iterator end,
- char sep) {
- string result;
- for (vector<Node*>::iterator i = begin; i != end; ++i) {
- if (!result.empty())
- result.push_back(sep);
- const string& path = (*i)->PathDecanonicalized();
+void EdgeEval::AppendPathList(std::string* out_append,
+ std::vector<Node*>::iterator begin,
+ std::vector<Node*>::iterator end,
+ char sep) {
+ for (auto it = begin; it != end; ++it) {
+ if (it != begin)
+ out_append->push_back(sep);
+
+ const string& path = (*it)->PathDecanonicalized();
if (escape_in_out_ == kShellEscape) {
#if _WIN32
- GetWin32EscapedString(path, &result);
+ GetWin32EscapedString(path, out_append);
#else
- GetShellEscapedString(path, &result);
+ GetShellEscapedString(path, out_append);
#endif
} else {
- result.append(path);
+ out_append->append(path);
}
}
- return result;
}
-string Edge::EvaluateCommand(bool incl_rsp_file) {
- string command = GetBinding("command");
+static const HashedStrView kCommand { "command" };
+static const HashedStrView kDepfile { "depfile" };
+static const HashedStrView kDyndep { "dyndep" };
+static const HashedStrView kRspfile { "rspfile" };
+static const HashedStrView kRspFileContent { "rspfile_content" };
+
+bool Edge::EvaluateCommand(std::string* out_append, bool incl_rsp_file,
+ std::string* err) {
+ METRIC_RECORD("eval command");
+ if (!EvaluateVariable(out_append, kCommand, err))
+ return false;
if (incl_rsp_file) {
- string rspfile_content = GetBinding("rspfile_content");
- if (!rspfile_content.empty())
- command += ";rspfile=" + rspfile_content;
+ std::string rspfile_content;
+ if (!EvaluateVariable(&rspfile_content, kRspFileContent, err))
+ return false;
+ if (!rspfile_content.empty()) {
+ out_append->append(";rspfile=");
+ out_append->append(rspfile_content);
+ }
}
+ return true;
+}
+
+std::string Edge::EvaluateCommand(bool incl_rsp_file) {
+ std::string command;
+ std::string err;
+ if (!EvaluateCommand(&command, incl_rsp_file, &err))
+ Fatal("%s", err.c_str());
return command;
}
-string Edge::GetBinding(const string& key) {
- EdgeEnv env(this, EdgeEnv::kShellEscape);
- return env.LookupVariable(key);
+static const HashedStrView kRestat { "restat" };
+static const HashedStrView kGenerator { "generator" };
+static const HashedStrView kDeps { "deps" };
+static const HashedStrView kPhonyOutput { "phony_output" };
+
+bool Edge::PrecomputeDepScanInfo(std::string* err) {
+ if (dep_scan_info_.valid)
+ return true;
+
+ // Precompute boolean flags.
+ auto get_bool_var = [this, err](const HashedStrView& var,
+ EdgeEval::EscapeKind escape, bool* out) {
+ std::string value;
+ if (!EvaluateVariable(&value, var, err, EdgeEval::kFinalScope, escape))
+ return false;
+ *out = !value.empty();
+ return true;
+ };
+ if (!get_bool_var(kRestat, EdgeEval::kShellEscape, &dep_scan_info_.restat)) return false;
+ if (!get_bool_var(kGenerator, EdgeEval::kShellEscape, &dep_scan_info_.generator)) return false;
+ if (!get_bool_var(kDeps, EdgeEval::kShellEscape, &dep_scan_info_.deps)) return false;
+ if (!get_bool_var(kDepfile, EdgeEval::kDoNotEscape, &dep_scan_info_.depfile)) return false;
+ if (!get_bool_var(kPhonyOutput, EdgeEval::kShellEscape, &dep_scan_info_.phony_output)) return false;
+
+ // Precompute the command hash.
+ std::string command;
+ if (!EvaluateCommand(&command, /*incl_rsp_file=*/true, err))
+ return false;
+ dep_scan_info_.command_hash = BuildLog::LogEntry::HashCommand(command);
+
+ dep_scan_info_.valid = true;
+ return true;
}
-bool Edge::GetBindingBool(const string& key) {
- return !GetBinding(key).empty();
+/// Returns dependency-scanning info or exits with a fatal error.
+const Edge::DepScanInfo& Edge::ComputeDepScanInfo() {
+ std::string err;
+ if (!PrecomputeDepScanInfo(&err))
+ Fatal("%s", err.c_str());
+ return dep_scan_info_;
}
-string Edge::GetUnescapedDepfile() {
- EdgeEnv env(this, EdgeEnv::kDoNotEscape);
- return env.LookupVariable("depfile");
+void Edge::SetRestat() {
+ std::string err;
+ if (!PrecomputeDepScanInfo(&err))
+ Fatal("%s", err.c_str());
+ dep_scan_info_.restat = true;
}
-string Edge::GetUnescapedDyndep() {
- EdgeEnv env(this, EdgeEnv::kDoNotEscape);
- return env.LookupVariable("dyndep");
+bool Edge::EvaluateVariable(std::string* out_append, const HashedStrView& key,
+ std::string* err, EdgeEval::EvalPhase phase,
+ EdgeEval::EscapeKind escape) {
+ EdgeEval eval(this, phase, escape);
+ return eval.EvaluateVariable(out_append, key, err);
}
-string Edge::GetUnescapedRspfile() {
- EdgeEnv env(this, EdgeEnv::kDoNotEscape);
- return env.LookupVariable("rspfile");
+std::string Edge::GetBindingImpl(const HashedStrView& key,
+ EdgeEval::EvalPhase phase,
+ EdgeEval::EscapeKind escape) {
+ std::string result;
+ std::string err;
+ if (!EvaluateVariable(&result, key, &err, phase, escape))
+ Fatal("%s", err.c_str());
+ return result;
+}
+
+std::string Edge::GetBinding(const HashedStrView& key) {
+ return GetBindingImpl(key, EdgeEval::kFinalScope, EdgeEval::kShellEscape);
+}
+
+std::string Edge::GetUnescapedDepfile() {
+ return GetBindingImpl(kDepfile, EdgeEval::kFinalScope, EdgeEval::kDoNotEscape);
+}
+
+std::string Edge::GetUnescapedDyndep() {
+ return GetBindingImpl(kDyndep, EdgeEval::kFinalScope, EdgeEval::kDoNotEscape);
+}
+
+std::string Edge::GetUnescapedRspfile() {
+ return GetBindingImpl(kRspfile, EdgeEval::kFinalScope, EdgeEval::kDoNotEscape);
}
void Edge::Dump(const char* prefix) const {
@@ -442,6 +722,13 @@
i != outputs_.end() && *i != NULL; ++i) {
printf("%s ", (*i)->path().c_str());
}
+ if (!validations_.empty()) {
+ printf(" validations ");
+ for (vector<Node*>::const_iterator i = validations_.begin();
+ i != validations_.end() && *i != NULL; ++i) {
+ printf("%s ", (*i)->path().c_str());
+ }
+ }
if (pool_) {
if (!pool_->name().empty()) {
printf("(in pool '%s')", pool_->name().c_str());
@@ -465,7 +752,21 @@
// of the form "build a: phony ... a ...". Restrict our
// "phonycycle" diagnostic option to the form it used.
return is_phony() && outputs_.size() == 1 && implicit_outs_ == 0 &&
- implicit_deps_ == 0;
+ implicit_deps_ == 0 && order_only_deps_ == 0;
+}
+
+bool Edge::EvaluateVariableSelfOnly(std::string* out_append,
+ const HashedStrView& var) const {
+ // ninja allows declaring the same binding repeatedly on an edge. Use the
+ // last matching binding.
+ const auto it_end = unevaled_bindings_.rend();
+ for (auto it = unevaled_bindings_.rbegin(); it != it_end; ++it) {
+ if (var == it->first) {
+ EvaluateBindingInScope(out_append, it->second, pos_.scope_pos());
+ return true;
+ }
+ }
+ return false;
}
// static
@@ -483,6 +784,66 @@
return result;
}
+Node::~Node() {
+ EdgeList* node = out_edges_.load();
+ while (node != nullptr) {
+ EdgeList* next = node->next;
+ delete node;
+ node = next;
+ }
+}
+
+// Does the node have at least one out edge?
+bool Node::has_out_edge() const {
+ return out_edges_.load() != nullptr;
+}
+
+std::vector<Edge*> Node::GetOutEdges() const {
+ // Include out-edges from the manifest.
+ std::vector<Edge*> result;
+ for (EdgeList* node = out_edges_.load(); node != nullptr; node = node->next) {
+ result.push_back(node->edge);
+ }
+ std::sort(result.begin(), result.end(), EdgeCmp());
+
+ // Add extra out-edges from depfiles and the deps log. Preserve the order
+ // of these extra edges; don't sort them.
+ std::copy(dep_scan_out_edges_.begin(), dep_scan_out_edges_.end(),
+ std::back_inserter(result));
+
+ return result;
+}
+
+std::vector<Edge*> Node::GetValidationOutEdges() const {
+ std::vector<Edge*> result;
+ for (EdgeList* node = validation_out_edges_.load(); node != nullptr; node = node->next) {
+ result.push_back(node->edge);
+ }
+ std::sort(result.begin(), result.end(), EdgeCmp());
+
+ return result;
+}
+
+void Node::AddOutEdge(Edge* edge) {
+ EdgeList* new_node = new EdgeList { edge };
+ while (true) {
+ EdgeList* cur_head = out_edges_.load();
+ new_node->next = cur_head;
+ if (out_edges_.compare_exchange_weak(cur_head, new_node))
+ break;
+ }
+}
+
+void Node::AddValidationOutEdge(Edge* edge) {
+ EdgeList* new_node = new EdgeList { edge };
+ while (true) {
+ EdgeList* cur_head = validation_out_edges_.load();
+ new_node->next = cur_head;
+ if (validation_out_edges_.compare_exchange_weak(cur_head, new_node))
+ break;
+ }
+}
+
void Node::Dump(const char* prefix) const {
printf("%s <%s 0x%p> mtime: %" PRId64 "%s, (:%s), ",
prefix, path().c_str(), this,
@@ -494,20 +855,31 @@
printf("no in-edge\n");
}
printf(" out edges:\n");
- for (vector<Edge*>::const_iterator e = out_edges().begin();
- e != out_edges().end() && *e != NULL; ++e) {
+ const std::vector<Edge*> out_edges = GetOutEdges();
+ for (vector<Edge*>::const_iterator e = out_edges.begin();
+ e != out_edges.end() && *e != NULL; ++e) {
(*e)->Dump(" +- ");
}
+ const std::vector<Edge*> validation_out_edges = GetValidationOutEdges();
+ if (!validation_out_edges.empty()) {
+ printf(" validation out edges:\n");
+ for (vector<Edge*>::const_iterator e = validation_out_edges.begin();
+ e != validation_out_edges.end() && *e != NULL; ++e) {
+ (*e)->Dump(" +- ");
+ }
+ }
}
bool ImplicitDepLoader::LoadDeps(Edge* edge, string* err) {
- string deps_type = edge->GetBinding("deps");
- if (!deps_type.empty())
+ if (edge->UsesDepsLog())
return LoadDepsFromLog(edge, err);
- string depfile = edge->GetUnescapedDepfile();
- if (!depfile.empty())
+ if (edge->UsesDepfile()) {
+ std::string depfile = edge->GetUnescapedDepfile();
+ assert(!depfile.empty() &&
+ "UsesDepfile was set, so the depfile should be non-empty");
return LoadDepFile(edge, depfile, err);
+ }
// No deps to load.
return true;
@@ -553,8 +925,7 @@
// Check that this depfile matches the edge's output, if not return false to
// mark the edge as dirty.
Node* first_output = edge->outputs_[0];
- StringPiece opath = StringPiece(first_output->path());
- if (opath != depfile.out_) {
+ if (first_output->path() != depfile.out_) {
EXPLAIN("expected depfile '%s' to mention '%s', got '%s'", path.c_str(),
first_output->path().c_str(), depfile.out_.AsString().c_str());
return false;
@@ -574,7 +945,7 @@
Node* node = state_->GetNode(*i, slash_bits);
*implicit_dep = node;
- node->AddOutEdge(edge);
+ node->AddOutEdgeDepScan(edge);
CreatePhonyInEdge(node);
}
@@ -602,7 +973,7 @@
for (int i = 0; i < deps->node_count; ++i, ++implicit_dep) {
Node* node = deps->nodes[i];
*implicit_dep = node;
- node->AddOutEdge(edge);
+ node->AddOutEdgeDepScan(edge);
CreatePhonyInEdge(node);
}
return true;
@@ -623,6 +994,7 @@
Edge* phony_edge = state_->AddEdge(&State::kPhonyRule);
node->set_in_edge(phony_edge);
phony_edge->outputs_.push_back(node);
+ ++phony_edge->explicit_outs_;
// RecomputeDirty might not be called for phony_edge if a previous call
// to RecomputeDirty had caused the file to be stat'ed. Because previous
@@ -631,4 +1003,6 @@
// to avoid a potential stuck build. If we do call RecomputeDirty for
// this node, it will simply set outputs_ready_ to the correct value.
phony_edge->outputs_ready_ = true;
+
+ phony_edge->phony_from_depfile_ = true;
}
diff --git a/src/graph.h b/src/graph.h
index 75edbc5..776c1ac 100644
--- a/src/graph.h
+++ b/src/graph.h
@@ -15,6 +15,8 @@
#ifndef NINJA_GRAPH_H_
#define NINJA_GRAPH_H_
+#include <atomic>
+#include <set>
#include <string>
#include <vector>
using namespace std;
@@ -32,33 +34,108 @@
struct Node;
struct Pool;
struct State;
+struct ThreadPool;
+
+/// A reference to a path from the lexer. This path is unevaluated, stored in
+/// mmap'ed memory, and is guaranteed to be followed by a terminating character
+/// (e.g. whitespace, a colon or pipe, etc).
+struct LexedPath {
+ StringPiece str_;
+};
+
+#ifdef _WIN32
+
+/// On Windows, we record the '/'-vs-'\' slash direction in the first reference
+/// to a path, which determines the path strings used in command-lines.
+struct NodeSlashBits {
+ NodeSlashBits() {}
+ NodeSlashBits(uint64_t value) : slash_bits_(value) {}
+ uint64_t slash_bits() const { return slash_bits_; }
+private:
+ uint64_t slash_bits_ = 0;
+};
+
+#else
+
+/// By default, '\' is not recognized as a path separator, and slash_bits is
+/// always 0.
+struct NodeSlashBits {
+ NodeSlashBits() {}
+ NodeSlashBits(uint64_t /*value*/) {}
+ uint64_t slash_bits() const { return 0; }
+};
+
+#endif // _WIN32
+
+/// Record the earliest reference to the node, which is needed for two uses:
+///
+/// - To verify that a node exists before a "default" declaration references
+/// it.
+/// - On Windows, the earliest reference to the node determines the slash_bits
+/// value for decanonicalizing the node's path.
+///
+/// On Windows, atomic operations on this struct probably use a spin lock, but
+/// it can be configured to use a DWCAS (e.g. -mcx16 with Clang/libc++). On
+/// other targets, the empty base class occupies 0 bytes, and atomic operations
+/// will be lock-free.
+struct NodeFirstReference : NodeSlashBits {
+ NodeFirstReference() {}
+ NodeFirstReference(DeclIndex loc, uint64_t slash_bits)
+ : NodeSlashBits(slash_bits), loc_(loc) {}
+ DeclIndex dfs_location() const { return loc_; }
+private:
+ DeclIndex loc_ = kLastDeclIndex;
+};
+
+inline bool operator<(const NodeFirstReference& x,
+ const NodeFirstReference& y) {
+ if (x.dfs_location() < y.dfs_location()) return true;
+ if (x.dfs_location() > y.dfs_location()) return false;
+ return x.slash_bits() < y.slash_bits();
+}
/// Information about a node in the dependency graph: the file, whether
/// it's dirty, mtime, etc.
struct Node {
- Node(const string& path, uint64_t slash_bits)
+ Node(const HashedStrView& path, uint64_t initial_slash_bits)
: path_(path),
- slash_bits_(slash_bits),
- mtime_(-1),
- dirty_(false),
- dyndep_pending_(false),
- in_edge_(NULL),
- id_(-1) {}
+ first_reference_({ kLastDeclIndex, initial_slash_bits }) {}
+ ~Node();
+
+ /// Precompute the node's Stat() call from a worker thread with exclusive
+ /// access to this node. Returns false on error.
+ bool PrecomputeStat(DiskInterface* disk_interface, string* err);
+
+ /// After the dependency scan is complete, reset the precomputed mtime so it
+ /// can't affect later StatIfNecessary() calls.
+ void ClearPrecomputedStat() {
+ precomputed_mtime_ = -1;
+ }
/// Return false on error.
+ /// Uses stat() or lstat() as appropriate.
bool Stat(DiskInterface* disk_interface, string* err);
+ /// Only use when lstat() is desired (output files)
+ bool LStat(DiskInterface* disk_interface, bool* is_dir, string* err);
+
/// Return false on error.
bool StatIfNecessary(DiskInterface* disk_interface, string* err) {
if (status_known())
return true;
+ if (precomputed_mtime_ >= 0) {
+ mtime_ = precomputed_mtime_;
+ return true;
+ }
return Stat(disk_interface, err);
}
/// Mark as not-yet-stat()ed and not dirty.
void ResetState() {
mtime_ = -1;
+ precomputed_mtime_ = -1;
dirty_ = false;
+ precomputed_dirtiness_ = false;
}
/// Mark the Node as already-stat()ed and missing.
@@ -74,14 +151,14 @@
return mtime_ != -1;
}
- const string& path() const { return path_; }
+ const std::string& path() const { return path_.str(); }
+ const HashedStr& path_hashed() const { return path_; }
/// Get |path()| but use slash_bits to convert back to original slash styles.
string PathDecanonicalized() const {
- return PathDecanonicalized(path_, slash_bits_);
+ return PathDecanonicalized(path_.str(), slash_bits());
}
static string PathDecanonicalized(const string& path,
uint64_t slash_bits);
- uint64_t slash_bits() const { return slash_bits_; }
TimeStamp mtime() const { return mtime_; }
@@ -89,6 +166,9 @@
void set_dirty(bool dirty) { dirty_ = dirty; }
void MarkDirty() { dirty_ = true; }
+ bool precomputed_dirtiness() const { return precomputed_dirtiness_; }
+ void set_precomputed_dirtiness(bool value) { precomputed_dirtiness_ = value; }
+
bool dyndep_pending() const { return dyndep_pending_; }
void set_dyndep_pending(bool pending) { dyndep_pending_ = pending; }
@@ -98,42 +178,126 @@
int id() const { return id_; }
void set_id(int id) { id_ = id; }
- const vector<Edge*>& out_edges() const { return out_edges_; }
- void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); }
+ // Thread-safe properties.
+ DeclIndex dfs_location() const {
+ return first_reference_.load().dfs_location();
+ }
+ uint64_t slash_bits() const {
+ return first_reference_.load().slash_bits();
+ }
+ void UpdateFirstReference(DeclIndex dfs_location, uint64_t slash_bits) {
+ AtomicUpdateMinimum(&first_reference_, { dfs_location, slash_bits });
+ }
+
+ // Thread-safe properties.
+ bool has_out_edge() const;
+ std::vector<Edge*> GetOutEdges() const;
+ std::vector<Edge*> GetValidationOutEdges() const;
+ void AddOutEdge(Edge* edge);
+ void AddValidationOutEdge(Edge* edge);
+
+ /// Add an out-edge from the dependency scan. This function differs from
+ /// AddOutEdge in several ways:
+ /// - It uses a simple vector, which is faster for single-threaded use.
+ /// - It's not thread-safe.
+ /// - It preserves edge order. Edges added with AddOutEdge come from the
+ /// manifest and are ordered by their position within the manifest
+ /// (represented with the edge ID). Dep scan edges, on the other hand,
+ /// are ordered by a DFS walk from target nodes to their dependencies.
+ /// (I'm not sure whether this order is practically important.)
+ void AddOutEdgeDepScan(Edge* edge) { dep_scan_out_edges_.push_back(edge); }
void Dump(const char* prefix="") const;
-private:
- string path_;
+ // Used in the inputs debug tool.
+ bool InputsChecked() const { return inputs_checked_; }
+ void MarkInputsChecked() { inputs_checked_ = true; }
- /// Set bits starting from lowest for backslashes that were normalized to
- /// forward slashes by CanonicalizePath. See |PathDecanonicalized|.
- uint64_t slash_bits_;
+private:
+ const HashedStr path_;
/// Possible values of mtime_:
/// -1: file hasn't been examined
/// 0: we looked, and file doesn't exist
/// >0: actual file's mtime
- TimeStamp mtime_;
+ TimeStamp mtime_ = -1;
+
+ /// If this value is >= 0, it represents a precomputed mtime for the node.
+ TimeStamp precomputed_mtime_ = -1;
/// Dirty is true when the underlying file is out-of-date.
/// But note that Edge::outputs_ready_ is also used in judging which
/// edges to build.
- bool dirty_;
+ bool dirty_ = false;
+
+ /// Set to true once the node's stat and command-hash info have been
+ /// precomputed.
+ bool precomputed_dirtiness_ = false;
/// Store whether dyndep information is expected from this node but
/// has not yet been loaded.
- bool dyndep_pending_;
-
- /// The Edge that produces this Node, or NULL when there is no
- /// known edge to produce it.
- Edge* in_edge_;
-
- /// All Edges that use this Node as an input.
- vector<Edge*> out_edges_;
+ bool dyndep_pending_ = false;
/// A dense integer id for the node, assigned and used by DepsLog.
- int id_;
+ int id_ = -1;
+
+ std::atomic<NodeFirstReference> first_reference_;
+
+ Edge* in_edge_ = nullptr;
+
+ struct EdgeList {
+ EdgeList(Edge* edge=nullptr, EdgeList* next=nullptr)
+ : edge(edge), next(next) {}
+
+ Edge* edge = nullptr;
+ EdgeList* next = nullptr;
+ };
+
+ /// All Edges that use this Node as an input. The order of this list is
+ /// non-deterministic. An accessor function sorts it each time it's used.
+ std::atomic<EdgeList*> out_edges_ { nullptr };
+ std::atomic<EdgeList*> validation_out_edges_ { nullptr };
+
+ std::vector<Edge*> dep_scan_out_edges_;
+
+ /// Stores if this node's inputs have been already computed. Used in the
+ /// inputs debug tool.
+ bool inputs_checked_ = false;
+};
+
+struct EdgeEval {
+ enum EvalPhase { kParseTime, kFinalScope };
+ enum EscapeKind { kShellEscape, kDoNotEscape };
+
+ EdgeEval(Edge* edge, EvalPhase eval_phase, EscapeKind escape)
+ : edge_(edge),
+ eval_phase_(eval_phase),
+ escape_in_out_(escape) {}
+
+ /// Looks up the variable and appends its value to the output buffer. Returns
+ /// false on error (i.e. a cycle in rule variable expansion).
+ bool EvaluateVariable(std::string* out_append, const HashedStrView& var,
+ std::string* err);
+
+ /// There are only a small number of bindings allowed on a rule. If we recurse
+ /// enough times, we're guaranteed to repeat a variable.
+ static constexpr int kEvalRecursionLimit = 16;
+
+private:
+ Edge* edge_ = nullptr;
+
+ EvalPhase eval_phase_ = kFinalScope;
+
+ /// The kind of escaping to do on $in and $out path variables.
+ EscapeKind escape_in_out_ = kShellEscape;
+
+ int recursion_count_ = 0;
+ StringPiece recursion_vars_[kEvalRecursionLimit];
+
+ void AppendPathList(std::string* out_append,
+ std::vector<Node*>::iterator begin,
+ std::vector<Node*>::iterator end,
+ char sep);
};
/// An edge in the dependency graph; links between Nodes using Rules.
@@ -144,10 +308,15 @@
VisitDone
};
- Edge() : rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL),
- mark_(VisitNone), outputs_ready_(false), deps_loaded_(false),
- deps_missing_(false), implicit_deps_(0), order_only_deps_(0),
- implicit_outs_(0) {}
+ struct DepScanInfo {
+ bool valid = false;
+ bool restat = false;
+ bool generator = false;
+ bool deps = false;
+ bool depfile = false;
+ bool phony_output = false;
+ uint64_t command_hash = 0;
+ };
/// Return true if all inputs' in-edges are ready.
bool AllInputsReady() const;
@@ -155,12 +324,48 @@
/// Expand all variables in a command and return it as a string.
/// If incl_rsp_file is enabled, the string will also contain the
/// full contents of a response file (if applicable)
- string EvaluateCommand(bool incl_rsp_file = false);
+ bool EvaluateCommand(std::string* out_append, bool incl_rsp_file,
+ std::string* err);
- /// Returns the shell-escaped value of |key|.
- string GetBinding(const string& key);
- bool GetBindingBool(const string& key);
+ /// Convenience method. This method must not be called from a worker thread,
+ /// because it could abort with a fatal error. (For consistency with other
+ /// Get*/Evaluate* methods, a better name might be GetCommand.)
+ std::string EvaluateCommand(bool incl_rsp_file = false);
+ /// Attempts to evaluate info needed for scanning dependencies.
+ bool PrecomputeDepScanInfo(std::string* err);
+
+ /// Returns dependency-scanning info or exits with a fatal error. These
+ /// methods must not be called until after the manifest has been loaded.
+ const DepScanInfo& ComputeDepScanInfo();
+ uint64_t GetCommandHash() { return ComputeDepScanInfo().command_hash; }
+ bool IsRestat() { return ComputeDepScanInfo().restat; }
+ bool IsGenerator() { return ComputeDepScanInfo().generator; }
+ bool IsPhonyOutput() { return ComputeDepScanInfo().phony_output; }
+ bool UsesDepsLog() { return ComputeDepScanInfo().deps; }
+ bool UsesDepfile() { return ComputeDepScanInfo().depfile; }
+
+ /// Dyndep can make an edge restat at runtime
+ void SetRestat();
+
+ /// Appends the value of |key| to the output buffer. On error, returns false,
+ /// and the content of the output buffer is unspecified.
+ bool EvaluateVariable(std::string* out_append, const HashedStrView& key,
+ std::string* err,
+ EdgeEval::EvalPhase phase=EdgeEval::kFinalScope,
+ EdgeEval::EscapeKind escape=EdgeEval::kShellEscape);
+
+private:
+ std::string GetBindingImpl(const HashedStrView& key,
+ EdgeEval::EvalPhase phase,
+ EdgeEval::EscapeKind escape);
+
+public:
+ /// Convenience method for EvaluateVariable. On failure, it issues a fatal
+ /// error. This function must not be called from a worker thread because:
+ /// - Error reporting should be deterministic, and
+ /// - Fatal() destructs static globals, which a worker thread could be using.
+ std::string GetBinding(const HashedStrView& key);
/// Like GetBinding("depfile"), but without shell escaping.
string GetUnescapedDepfile();
/// Like GetBinding("dyndep"), but without shell escaping.
@@ -170,17 +375,47 @@
void Dump(const char* prefix="") const;
- const Rule* rule_;
- Pool* pool_;
+ /// Temporary fields used only during manifest parsing.
+ struct DeferredPathList {
+ enum Type {
+ INPUT = 0,
+ OUTPUT = 1,
+ VALIDATION = 2,
+ };
+
+ DeferredPathList(const char* lexer_pos=nullptr,
+ Type type = INPUT, int count=0)
+ : lexer_pos(lexer_pos), type(type), count(count) {}
+
+ const char* lexer_pos = nullptr;
+ Type type;
+ int count = 0;
+ };
+ struct {
+ StringPiece rule_name;
+ size_t rule_name_diag_pos = 0;
+ size_t final_diag_pos = 0;
+ std::vector<DeferredPathList> deferred_path_lists;
+ } parse_state_;
+
+ RelativePosition pos_;
+ const Rule* rule_ = nullptr;
+ Pool* pool_ = nullptr;
vector<Node*> inputs_;
vector<Node*> outputs_;
- Node* dyndep_;
- BindingEnv* env_;
- VisitMark mark_;
- bool outputs_ready_;
- bool deps_loaded_;
- bool deps_missing_;
+ vector<Node*> validations_;
+ Node* dyndep_ = nullptr;
+ std::vector<std::pair<HashedStr, std::string>> unevaled_bindings_;
+ VisitMark mark_ = VisitNone;
+ bool precomputed_dirtiness_ = false;
+ size_t id_ = 0;
+ bool outputs_ready_ = false;
+ bool deps_loaded_ = false;
+ bool deps_missing_ = false;
+ bool phony_from_depfile_ = false;
+ DepScanInfo dep_scan_info_;
+ DeclIndex dfs_location() const { return pos_.dfs_location(); }
const Rule& rule() const { return *rule_; }
Pool* pool() const { return pool_; }
int weight() const { return 1; }
@@ -194,8 +429,9 @@
// don't cause the target to rebuild.
// These are stored in inputs_ in that order, and we keep counts of
// #2 and #3 when we need to access the various subsets.
- int implicit_deps_;
- int order_only_deps_;
+ int explicit_deps_ = 0;
+ int implicit_deps_ = 0;
+ int order_only_deps_ = 0;
bool is_implicit(size_t index) {
return index >= inputs_.size() - order_only_deps_ - implicit_deps_ &&
!is_order_only(index);
@@ -209,16 +445,32 @@
// 2) implicit outs, which the target generates but are not part of $out.
// These are stored in outputs_ in that order, and we keep a count of
// #2 to use when we need to access the various subsets.
- int implicit_outs_;
+ int explicit_outs_ = 0;
+ int implicit_outs_ = 0;
bool is_implicit_out(size_t index) const {
return index >= outputs_.size() - implicit_outs_;
}
+ int validation_deps_ = 0;
+
bool is_phony() const;
bool use_console() const;
bool maybe_phonycycle_diagnostic() const;
+
+ /// Search for a binding on this edge and append its value to the output
+ /// string. Does not search the enclosing scope. Use other functions to
+ /// include ancestor scopes and rule bindings.
+ bool EvaluateVariableSelfOnly(std::string* out_append,
+ const HashedStrView& var) const;
};
+struct EdgeCmp {
+ bool operator()(const Edge* a, const Edge* b) const {
+ return a->id_ < b->id_;
+ }
+};
+
+typedef set<Edge*, EdgeCmp> EdgeSet;
/// ImplicitDepLoader loads implicit dependencies, as referenced via the
/// "depfile" attribute in build files.
@@ -268,18 +520,31 @@
struct DependencyScan {
DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log,
DiskInterface* disk_interface,
- DepfileParserOptions const* depfile_parser_options)
+ DepfileParserOptions const* depfile_parser_options,
+ bool missing_phony_is_err)
: build_log_(build_log),
disk_interface_(disk_interface),
dep_loader_(state, deps_log, disk_interface, depfile_parser_options),
- dyndep_loader_(state, disk_interface) {}
+ dyndep_loader_(state, disk_interface),
+ missing_phony_is_err_(missing_phony_is_err) {}
- /// Update the |dirty_| state of the given node by inspecting its input edge.
+ /// Used for tests.
+ bool RecomputeDirty(Node* node, std::vector<Node*>* validation_nodes,
+ std::string* err) {
+ std::vector<Node*> nodes = {node};
+ return RecomputeNodesDirty(nodes, validation_nodes, err);
+ }
+
+ /// Update the |dirty_| state of the given nodes by transitively inspecting
+ /// their input edges.
/// Examine inputs, outputs, and command lines to judge whether an edge
/// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_|
/// state accordingly.
+ /// Appends any validation nodes found to the nodes parameter.
/// Returns false on failure.
- bool RecomputeDirty(Node* node, string* err);
+ bool RecomputeNodesDirty(const std::vector<Node*>& initial_nodes,
+ std::vector<Node*>* validation_nodes,
+ std::string* err);
/// Recompute whether any output of the edge is dirty, if so sets |*dirty|.
/// Returns false on failure.
@@ -305,18 +570,33 @@
bool LoadDyndeps(Node* node, DyndepFile* ddf, string* err) const;
private:
- bool RecomputeDirty(Node* node, vector<Node*>* stack, string* err);
+ /// Find the transitive closure of edges and nodes that the given node depends
+ /// on. Each Node and Edge is guaranteed to appear at most once in an output
+ /// vector. The returned lists are not guaranteed to be a superset or a subset
+ /// of the nodes and edges that RecomputeNodesDirty will initialize.
+ void CollectPrecomputeLists(Node* node, std::vector<Node*>* nodes,
+ std::vector<Edge*>* edges);
+
+ bool PrecomputeNodesDirty(const std::vector<Node*>& nodes,
+ const std::vector<Edge*>& edges,
+ ThreadPool* thread_pool, std::string* err);
+
+ bool RecomputeNodeDirty(Node* node, vector<Node*>* stack,
+ vector<Node*>* validation_nodes, string* err);
+
bool VerifyDAG(Node* node, vector<Node*>* stack, string* err);
/// Recompute whether a given single output should be marked dirty.
/// Returns true if so.
bool RecomputeOutputDirty(Edge* edge, Node* most_recent_input,
- const string& command, Node* output);
+ uint64_t command_hash, Node* output);
BuildLog* build_log_;
DiskInterface* disk_interface_;
ImplicitDepLoader dep_loader_;
DyndepLoader dyndep_loader_;
+
+ bool missing_phony_is_err_;
};
#endif // NINJA_GRAPH_H_
diff --git a/src/graph_test.cc b/src/graph_test.cc
index c8cca1c..a7b9acb 100644
--- a/src/graph_test.cc
+++ b/src/graph_test.cc
@@ -18,7 +18,7 @@
#include "test.h"
struct GraphTest : public StateTestWithBuiltinRules {
- GraphTest() : scan_(&state_, NULL, NULL, &fs_, NULL) {}
+ GraphTest() : scan_(&state_, NULL, NULL, &fs_, NULL, false) {}
VirtualFileSystem fs_;
DependencyScan scan_;
@@ -31,7 +31,7 @@
fs_.Create("out", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
// A missing implicit dep *should* make the output dirty.
@@ -49,7 +49,7 @@
fs_.Create("implicit", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
// A modified implicit dep should make the output dirty.
@@ -69,7 +69,7 @@
fs_.Create("implicit.h", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
// implicit.h has changed, though our depfile refers to it with a
@@ -92,7 +92,7 @@
fs_.Create("data", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
// We have both an implicit and an explicit dep on implicit.h.
@@ -120,7 +120,7 @@
fs_.Create("out", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("out")->dirty());
@@ -136,7 +136,7 @@
fs_.Create("out", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("out")->dirty());
@@ -160,7 +160,7 @@
fs_.Create("in", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("out.imp")->dirty());
@@ -174,7 +174,7 @@
fs_.Create("in", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("out.imp")->dirty());
@@ -191,7 +191,7 @@
fs_.Create("out.o", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_FALSE(GetNode("out.o")->dirty());
@@ -239,7 +239,7 @@
fs_.Create("out.o", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_FALSE(GetNode("out.o")->dirty());
@@ -259,13 +259,13 @@
fs_.Create("out.o", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_FALSE(GetNode("out.o")->dirty());
state_.Reset();
fs_.RemoveFile("out.o.d");
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("out.o")->dirty());
}
@@ -312,7 +312,7 @@
"build n2: phony n1\n"
);
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("n2"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("n2"), NULL, &err));
ASSERT_EQ("", err);
Plan plan_;
@@ -331,10 +331,190 @@
parser_opts);
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err));
ASSERT_EQ("dependency cycle: a -> a [-w phonycycle=err]", err);
}
+TEST_F(GraphTest, OutputSymlinkSourceUpdate) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build other: cat\n"
+"build sym: cat\n"
+"build out: cat | sym\n"));
+
+ fs_.Create("out", "");
+ fs_.CreateSymlink("sym", "other");
+ fs_.Tick();
+ fs_.Create("other", "");
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, OutputSymlinkDangling) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build sym: cat\n"
+"build out: cat | sym\n"));
+
+ fs_.CreateSymlink("sym", "dangling");
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, InputSymlink) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat sym\n"));
+
+ fs_.Create("out", "");
+ fs_.CreateSymlink("sym", "in");
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, InputSymlinkUpdate) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat sym\n"));
+
+ fs_.Create("out", "");
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.CreateSymlink("sym", "in");
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ // This can be incorrect if the destination of the symlink changed to
+ // a file with an equal or older timestamp.
+ EXPECT_FALSE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, InputSymlinkDangling) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat sym\n"));
+
+ fs_.Create("out", "");
+ fs_.CreateSymlink("sym", "in");
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, InputDirectoryUpToDate) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat inputs\n"));
+
+ fs_.Create("out", "");
+ EXPECT_TRUE(fs_.MakeDir("inputs"));
+ EXPECT_TRUE(fs_.WriteFile("inputs/foo", ""));
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, InputDirectoryChanged) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat inputs\n"));
+
+ fs_.Create("out", "");
+ EXPECT_TRUE(fs_.MakeDir("inputs"));
+ fs_.Tick();
+ EXPECT_TRUE(fs_.WriteFile("inputs/foo", ""));
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, PhonyOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build foo: phony_out\n"));
+
+ Node* node = state_.LookupNode("foo");
+ Edge* edge = node->in_edge();
+ ASSERT_TRUE(edge->IsPhonyOutput());
+}
+
+TEST_F(GraphTest, PhonyOutputDependsOnPhonyOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build foo: phony_out\n"
+"build bar: phony_out foo\n"));
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("bar"), NULL, &err));
+ ASSERT_EQ("", err);
+}
+
+TEST_F(GraphTest, RealDependsOnPhonyOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"rule touch\n"
+" command = touch ${out}\n"
+"build foo: phony_out\n"
+"build bar: touch foo\n"));
+
+ string err;
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("bar"), NULL, &err));
+ EXPECT_EQ("real file 'bar' depends on phony output 'foo'\n", err);
+}
+
+TEST_F(GraphTest, PhonyDependsOnPhonyOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build foo: phony_out\n"
+"build bar: phony foo\n"));
+
+ string err;
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("bar"), NULL, &err));
+ EXPECT_EQ("real file 'bar' depends on phony output 'foo'\n", err);
+}
+
+TEST_F(GraphTest, MissingPhonyWithPhonyOutputs) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build foo: phony\n"));
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("foo"), NULL, &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(GetNode("foo")->dirty());
+
+ state_.Reset();
+ DependencyScan scan(&state_, NULL, NULL, &fs_, NULL, true);
+ EXPECT_FALSE(scan.RecomputeDirty(GetNode("foo"), NULL, &err));
+ EXPECT_EQ("output foo of phony edge doesn't exist. Missing 'phony_output = true'?", err);
+}
+
TEST_F(GraphTest, DependencyCycle) {
AssertParse(&state_,
"build out: cat mid\n"
@@ -343,7 +523,7 @@
"build pre: cat out\n");
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err);
}
@@ -351,7 +531,7 @@
string err;
AssertParse(&state_,
"build a b: cat a\n");
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err));
ASSERT_EQ("dependency cycle: a -> a", err);
}
@@ -359,7 +539,7 @@
string err;
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build b a: cat a\n"));
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err));
ASSERT_EQ("dependency cycle: a -> a", err);
}
@@ -368,7 +548,7 @@
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build a b: cat c\n"
"build c: cat a\n"));
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err));
ASSERT_EQ("dependency cycle: a -> c -> a", err);
}
@@ -380,7 +560,7 @@
"build b: cat a\n"
"build a e: cat d\n"
"build f: cat e\n"));
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("f"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("f"), NULL, &err));
ASSERT_EQ("dependency cycle: a -> d -> c -> b -> a", err);
}
@@ -396,7 +576,7 @@
fs_.Create("dep.d", "a: b\n");
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err));
ASSERT_EQ("dependency cycle: b -> b", err);
// Despite the depfile causing edge to be a cycle (it has outputs a and b,
@@ -421,7 +601,7 @@
fs_.Create("dep.d", "a: c\n");
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err));
ASSERT_EQ("dependency cycle: b -> c -> b", err);
// Despite the depfile causing edge to be a cycle (|edge| has outputs a and b,
@@ -448,7 +628,7 @@
fs_.Create("dep.d", "a: c\n");
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("d"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("d"), NULL, &err));
ASSERT_EQ("dependency cycle: b -> c -> b", err);
// Despite the depfile causing edge to be a cycle (|edge| has outputs a and b,
@@ -506,7 +686,7 @@
EXPECT_EQ("dd", edge->inputs_[1]->path());
EXPECT_EQ(0u, edge->implicit_deps_);
EXPECT_EQ(1u, edge->order_only_deps_);
- EXPECT_FALSE(edge->GetBindingBool("restat"));
+ EXPECT_FALSE(edge->IsRestat());
}
TEST_F(GraphTest, DyndepLoadMissingFile) {
@@ -641,11 +821,11 @@
EXPECT_EQ("dd", edge1->inputs_[2]->path());
EXPECT_EQ(1u, edge1->implicit_deps_);
EXPECT_EQ(1u, edge1->order_only_deps_);
- EXPECT_FALSE(edge1->GetBindingBool("restat"));
+ EXPECT_FALSE(edge1->IsRestat());
EXPECT_EQ(edge1, GetNode("out1imp")->in_edge());
Node* in1imp = GetNode("in1imp");
- ASSERT_EQ(1u, in1imp->out_edges().size());
- EXPECT_EQ(edge1, in1imp->out_edges()[0]);
+ ASSERT_EQ(1u, in1imp->GetOutEdges().size());
+ EXPECT_EQ(edge1, in1imp->GetOutEdges()[0]);
Edge* edge2 = GetNode("out2")->in_edge();
ASSERT_EQ(1u, edge2->outputs_.size());
@@ -657,10 +837,10 @@
EXPECT_EQ("dd", edge2->inputs_[2]->path());
EXPECT_EQ(1u, edge2->implicit_deps_);
EXPECT_EQ(1u, edge2->order_only_deps_);
- EXPECT_TRUE(edge2->GetBindingBool("restat"));
+ EXPECT_TRUE(edge2->IsRestat());
Node* in2imp = GetNode("in2imp");
- ASSERT_EQ(1u, in2imp->out_edges().size());
- EXPECT_EQ(edge2, in2imp->out_edges()[0]);
+ ASSERT_EQ(1u, in2imp->GetOutEdges().size());
+ EXPECT_EQ(edge2, in2imp->GetOutEdges()[0]);
}
TEST_F(GraphTest, DyndepFileMissing) {
@@ -672,7 +852,7 @@
);
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("loading 'dd': No such file or directory", err);
}
@@ -688,7 +868,7 @@
);
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("'out' not mentioned in its dyndep file 'dd'", err);
}
@@ -708,7 +888,7 @@
fs_.Create("in", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_FALSE(GetNode("in")->dirty());
@@ -736,7 +916,7 @@
fs_.Create("in", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_FALSE(GetNode("in")->dirty());
@@ -761,7 +941,7 @@
fs_.Create("out", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("dd")->dirty());
@@ -787,7 +967,7 @@
fs_.Create("out", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_FALSE(GetNode("dd")->dirty());
@@ -815,7 +995,7 @@
fs_.Create("out", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("dd1")->dirty());
@@ -844,7 +1024,7 @@
Edge* edge = GetNode("out")->in_edge();
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
EXPECT_EQ("dependency cycle: circ -> in -> circ", err);
// Verify that "out.d" was loaded exactly once despite
@@ -856,3 +1036,68 @@
EXPECT_EQ(1u, edge->implicit_deps_);
EXPECT_EQ(1u, edge->order_only_deps_);
}
+
+TEST_F(GraphTest, EdgeVarEvalPhase) {
+ // Variable lookups on edges can happen in one of two phases:
+ // - Parse-time: Only bindings declared before the reference are visible.
+ // - Final-scope: All bindings in the current scope (and ancestor scopes)
+ // are visible.
+ //
+ // An edge's pool is determined at parse-time, while most other bindings are
+ // looked up after manifest parsing is finished.
+
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"foo = A\n"
+"bar = B\n"
+"pool A\n"
+" depth = 3\n"
+"pool C\n"
+" depth = 3\n"
+"rule echo\n"
+" command = replaced by next line\n"
+" command = echo $foo,$bar\n"
+" pool = $foo\n"
+"build a: echo\n"
+" bar = replaced by next line\n"
+" bar = edge:$foo\n"
+"foo = C\n"
+"bar = D\n"));
+
+ Edge* edge = GetNode("a")->in_edge();
+ EXPECT_EQ("echo C,edge:A", edge->GetBinding("command"));
+ EXPECT_EQ("A", edge->pool()->name());
+ EXPECT_EQ("C", edge->GetBinding("pool"));
+}
+
+TEST_F(GraphTest, PhonyOutputAlwaysDirty) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build foo: phony_out\n"));
+
+ fs_.Create("foo", "");
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("foo"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(GetNode("foo")->dirty());
+}
+
+TEST_F(GraphTest, Validation) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat in |@ validate\n"
+"build validate: cat in\n"));
+
+ fs_.Create("in", "");
+ string err;
+ std::vector<Node*> validation_nodes;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &validation_nodes, &err));
+ ASSERT_EQ("", err);
+
+ ASSERT_EQ(validation_nodes.size(), 1);
+ EXPECT_EQ(validation_nodes[0]->path(), "validate");
+
+ EXPECT_TRUE(GetNode("out")->dirty());
+ EXPECT_TRUE(GetNode("validate")->dirty());
+}
diff --git a/src/graphviz.h b/src/graphviz.h
index 601c9b2..3a3282e 100644
--- a/src/graphviz.h
+++ b/src/graphviz.h
@@ -18,6 +18,7 @@
#include <set>
#include "dyndep.h"
+#include "graph.h"
struct DiskInterface;
struct Node;
@@ -34,7 +35,7 @@
DyndepLoader dyndep_loader_;
std::set<Node*> visited_nodes_;
- std::set<Edge*> visited_edges_;
+ EdgeSet visited_edges_;
};
#endif // NINJA_GRAPHVIZ_H_
diff --git a/src/hash_map.h b/src/hash_map.h
index 55d2c9d..717a7c9 100644
--- a/src/hash_map.h
+++ b/src/hash_map.h
@@ -12,12 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#ifndef NINJA_MAP_H_
-#define NINJA_MAP_H_
+#ifndef NINJA_HASH_MAP_H_
+#define NINJA_HASH_MAP_H_
-#include <algorithm>
#include <string.h>
-#include "string_piece.h"
#include "util.h"
// MurmurHash2, by Austin Appleby
@@ -53,71 +51,4 @@
return h;
}
-#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900)
-#include <unordered_map>
-
-namespace std {
-template<>
-struct hash<StringPiece> {
- typedef StringPiece argument_type;
- typedef size_t result_type;
-
- size_t operator()(StringPiece key) const {
- return MurmurHash2(key.str_, key.len_);
- }
-};
-}
-
-#elif defined(_MSC_VER)
-#include <hash_map>
-
-using stdext::hash_map;
-using stdext::hash_compare;
-
-struct StringPieceCmp : public hash_compare<StringPiece> {
- size_t operator()(const StringPiece& key) const {
- return MurmurHash2(key.str_, key.len_);
- }
- bool operator()(const StringPiece& a, const StringPiece& b) const {
- int cmp = memcmp(a.str_, b.str_, min(a.len_, b.len_));
- if (cmp < 0) {
- return true;
- } else if (cmp > 0) {
- return false;
- } else {
- return a.len_ < b.len_;
- }
- }
-};
-
-#else
-#include <ext/hash_map>
-
-using __gnu_cxx::hash_map;
-
-namespace __gnu_cxx {
-template<>
-struct hash<StringPiece> {
- size_t operator()(StringPiece key) const {
- return MurmurHash2(key.str_, key.len_);
- }
-};
-}
-#endif
-
-/// A template for hash_maps keyed by a StringPiece whose string is
-/// owned externally (typically by the values). Use like:
-/// ExternalStringHash<Foo*>::Type foos; to make foos into a hash
-/// mapping StringPiece => Foo*.
-template<typename V>
-struct ExternalStringHashMap {
-#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900)
- typedef std::unordered_map<StringPiece, V> Type;
-#elif defined(_MSC_VER)
- typedef hash_map<StringPiece, V, StringPieceCmp> Type;
-#else
- typedef hash_map<StringPiece, V> Type;
-#endif
-};
-
-#endif // NINJA_MAP_H_
+#endif // NINJA_HASH_MAP_H_
diff --git a/src/hashed_str_view.h b/src/hashed_str_view.h
new file mode 100644
index 0000000..ea19a4f
--- /dev/null
+++ b/src/hashed_str_view.h
@@ -0,0 +1,163 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_HASHED_STR_VIEW_
+#define NINJA_HASHED_STR_VIEW_
+
+#include <stdint.h>
+
+#include "string_piece.h"
+
+class HashedStr;
+
+using StrHashType = size_t;
+
+class HashedStrView {
+public:
+ constexpr HashedStrView() {}
+
+ HashedStrView(const char* str) : str_(str), hash_(HashStr(str_)) {}
+ HashedStrView(const std::string& str) : str_(str), hash_(HashStr(str_)) {}
+ HashedStrView(StringPiece str) : str_(str), hash_(HashStr(str_)) {}
+ inline HashedStrView(const HashedStr& str);
+
+ // Bypass the hashing step when necessary for better performance.
+ explicit HashedStrView(const StringPiece& str, StrHashType hash)
+ : str_(str), hash_(hash) {}
+
+ const StringPiece& str_view() const { return str_; }
+
+ StrHashType hash() const { return hash_; }
+ const char* data() const { return str_.data(); }
+ size_t size() const { return str_.size(); }
+ bool empty() const { return str_.empty(); }
+
+private:
+ // Reversing the order of these fields would break the constructors, most of
+ // which initialize hash_ using str_.
+ StringPiece str_;
+ StrHashType hash_ = 0;
+};
+
+class HashedStr {
+public:
+ HashedStr() {}
+
+ HashedStr(const char* str) : str_(str), hash_(HashStr(str_)) {}
+ HashedStr(const std::string& str) : str_(str), hash_(HashStr(str_)) {}
+ HashedStr(std::string&& str) : str_(std::move(str)), hash_(HashStr(str_)) {}
+ explicit HashedStr(StringPiece str) : str_(NonNullData(str), str.size()), hash_(HashStr(str_)) {}
+ explicit HashedStr(const HashedStrView& str) : str_(NonNullData(str.str_view()), str.size()), hash_(str.hash()) {}
+
+ HashedStr& operator=(std::string&& str) {
+ str_ = std::move(str);
+ hash_ = HashStr(str_);
+ return *this;
+ }
+
+ HashedStr& operator=(const std::string& str) {
+ str_ = str;
+ hash_ = HashStr(str_);
+ return *this;
+ }
+
+ HashedStr& operator=(StringPiece str) {
+ str_.assign(NonNullData(str), str.size());
+ hash_ = HashStr(str_);
+ return *this;
+ }
+
+ HashedStr& operator=(const HashedStrView& str) {
+ str_.assign(NonNullData(str.str_view()), str.size());
+ hash_ = str.hash();
+ return *this;
+ }
+
+ const std::string& str() const { return str_; }
+
+ StrHashType hash() const { return hash_; }
+ const char* c_str() const { return str_.c_str(); }
+ const char* data() const { return str_.data(); }
+ size_t size() const { return str_.size(); }
+ bool empty() const { return str_.empty(); }
+
+private:
+ static const char* NonNullData(const StringPiece& piece) {
+ return piece.data() == nullptr ? "" : piece.data();
+ }
+
+ // Reversing the order of these fields would break the constructors, most of
+ // which initialize hash_ using str_.
+ std::string str_;
+ StrHashType hash_ = 0;
+};
+
+inline HashedStrView::HashedStrView(const HashedStr& str) : str_(str.str()), hash_(str.hash()) {}
+
+inline bool operator==(const HashedStr& x, const HashedStr& y) {
+ return x.hash() == y.hash() && x.str() == y.str();
+}
+
+inline bool operator==(const HashedStr& x, const HashedStrView& y) {
+ return x.hash() == y.hash() && x.str() == y.str_view();
+}
+
+inline bool operator==(const HashedStrView& x, const HashedStr& y) {
+ return x.hash() == y.hash() && x.str_view() == y.str();
+}
+
+inline bool operator==(const HashedStrView& x, const HashedStrView& y) {
+ return x.hash() == y.hash() && x.str_view() == y.str_view();
+}
+
+inline bool operator!=(const HashedStr& x, const HashedStr& y) { return !(x == y); }
+inline bool operator!=(const HashedStr& x, const HashedStrView& y) { return !(x == y); }
+inline bool operator!=(const HashedStrView& x, const HashedStr& y) { return !(x == y); }
+inline bool operator!=(const HashedStrView& x, const HashedStrView& y) { return !(x == y); }
+
+namespace std {
+ template<> struct hash<HashedStrView> {
+ size_t operator()(const HashedStrView& x) const { return x.hash(); }
+ };
+
+ template<> struct hash<HashedStr> {
+ size_t operator()(const HashedStr& x) const { return x.hash(); }
+ };
+}
+
+// The comparison operators ignore the hash code. For efficiency, the
+// ConcurrentHashMap orders keys by their hash code first, then by their
+// operator< functions.
+
+inline bool operator<(const HashedStr& x, const HashedStr& y) { return x.str() < y.str(); }
+inline bool operator>(const HashedStr& x, const HashedStr& y) { return x.str() > y.str(); }
+inline bool operator<=(const HashedStr& x, const HashedStr& y) { return x.str() <= y.str(); }
+inline bool operator>=(const HashedStr& x, const HashedStr& y) { return x.str() >= y.str(); }
+
+inline bool operator<(const HashedStr& x, const HashedStrView& y) { return x.str() < y.str_view(); }
+inline bool operator>(const HashedStr& x, const HashedStrView& y) { return x.str() > y.str_view(); }
+inline bool operator<=(const HashedStr& x, const HashedStrView& y) { return x.str() <= y.str_view(); }
+inline bool operator>=(const HashedStr& x, const HashedStrView& y) { return x.str() >= y.str_view(); }
+
+inline bool operator<(const HashedStrView& x, const HashedStr& y) { return x.str_view() < y.str(); }
+inline bool operator>(const HashedStrView& x, const HashedStr& y) { return x.str_view() > y.str(); }
+inline bool operator<=(const HashedStrView& x, const HashedStr& y) { return x.str_view() <= y.str(); }
+inline bool operator>=(const HashedStrView& x, const HashedStr& y) { return x.str_view() >= y.str(); }
+
+inline bool operator<(const HashedStrView& x, const HashedStrView& y) { return x.str_view() < y.str_view(); }
+inline bool operator>(const HashedStrView& x, const HashedStrView& y) { return x.str_view() > y.str_view(); }
+inline bool operator<=(const HashedStrView& x, const HashedStrView& y) { return x.str_view() <= y.str_view(); }
+inline bool operator>=(const HashedStrView& x, const HashedStrView& y) { return x.str_view() >= y.str_view(); }
+
+#endif // NINJA_HASHED_STR_VIEW_
diff --git a/src/lexer.cc b/src/lexer.cc
index 35ae97b..a5c3c27 100644
--- a/src/lexer.cc
+++ b/src/lexer.cc
@@ -1,4 +1,4 @@
-/* Generated by re2c 0.16 */
+/* Generated by re2c 1.3 */
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,24 +18,79 @@
#include <stdio.h>
#include "eval_env.h"
+#include "graph.h"
#include "util.h"
-bool Lexer::Error(const string& message, string* err) {
+size_t AdvanceToNextManifestChunk(StringPiece content, size_t idx) {
+ assert(idx <= content.size());
+
+ // Iterate over each LF in the manifest, starting at the given index.
+ while (true) {
+ const void* next_line = memchr(content.data() + idx, '\n',
+ content.size() - idx);
+ if (next_line == nullptr) {
+ break;
+ }
+ idx = static_cast<const char*>(next_line) - content.data();
+ ++idx; // step over the LF
+
+ // The line must not be preceded by a line continuator. This logic can
+ // filter out more split candidates than strictly necessary:
+ // - The preceding line could have a comment that ends with a "$": "# $\n"
+ // - The preceding line could end with an escaped-dollar: "X=$$\n"
+ if ((idx >= 2 && content.substr(idx - 2, 2) == "$\n") ||
+ (idx >= 3 && content.substr(idx - 3, 3) == "$\r\n")) {
+ continue;
+ }
+
+ // Skip an indented line or a comment line, either of which could be part of
+ // an earlier declaration. Ninja allows unindented comments (as well as
+ // indented comments) inside a binding block, e.g.:
+ //
+ // build foo: cc
+ // # comment-line
+ // pool = link_pool
+ //
+ // Ninja doesn't allow blank lines in a binding block. This code could
+ // probably allow a chunk to start with a blank line, but it seems better if
+ // it doesn't.
+ if (idx >= content.size() ||
+ content[idx] == ' ' || content[idx] == '#' ||
+ content[idx] == '\r' || content[idx] == '\n') {
+ continue;
+ }
+
+ return idx;
+ }
+
+ return content.size();
+}
+
+bool DecorateErrorWithLocation(const std::string& filename,
+ const char* file_start,
+ size_t file_offset,
+ const std::string& message,
+ std::string* err) {
+ // Make a copy in case message and err alias.
+ std::string message_tmp = message;
+
// Compute line/column.
int line = 1;
- const char* line_start = input_.str_;
- for (const char* p = input_.str_; p < last_token_; ++p) {
+ const char* line_start = file_start;
+ const char* file_pos = file_start + file_offset;
+
+ for (const char* p = line_start; p < file_pos; ++p) {
if (*p == '\n') {
++line;
line_start = p + 1;
}
}
- int col = last_token_ ? (int)(last_token_ - line_start) : 0;
+ int col = (int)(file_pos - line_start);
char buf[1024];
- snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line);
+ snprintf(buf, sizeof(buf), "%s:%d: ", filename.c_str(), line);
*err = buf;
- *err += message + "\n";
+ *err += message_tmp + "\n";
// Add some context to the message.
const int kTruncateColumn = 72;
@@ -59,15 +114,16 @@
return false;
}
-Lexer::Lexer(const char* input) {
- Start("input", input);
+bool Lexer::Error(const std::string& message, std::string* err) {
+ return DecorateErrorWithLocation(filename_, input_.data(),
+ GetLastTokenOffset(), message, err);
}
-void Lexer::Start(StringPiece filename, StringPiece input) {
- filename_ = filename;
- input_ = input;
- ofs_ = input_.str_;
- last_token_ = NULL;
+bool Lexer::UnexpectedNulError(const char* pos, std::string* err) {
+ assert(*pos == '\0');
+ const char* msg = (pos == EndOfFile()) ? "unexpected EOF"
+ : "unexpected NUL byte";
+ return Error(msg, err);
}
const char* Lexer::TokenName(Token t) {
@@ -83,9 +139,11 @@
case NEWLINE: return "newline";
case PIPE2: return "'||'";
case PIPE: return "'|'";
+ case PIPEAT: return "'|@'";
case POOL: return "'pool'";
case RULE: return "'rule'";
case SUBNINJA: return "'subninja'";
+ case TNUL: return "nul byte";
case TEOF: return "eof";
}
return NULL; // not reached
@@ -117,6 +175,7 @@
Lexer::Token Lexer::ReadToken() {
const char* p = ofs_;
const char* q;
+ const char* r;
const char* start;
Lexer::Token token;
for (;;) {
@@ -127,7 +186,7 @@
unsigned int yyaccept = 0;
static const unsigned char yybm[] = {
0, 128, 128, 128, 128, 128, 128, 128,
- 128, 128, 0, 128, 128, 128, 128, 128,
+ 128, 128, 0, 128, 128, 0, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
160, 128, 128, 128, 128, 128, 128, 128,
@@ -219,7 +278,7 @@
}
yy2:
++p;
- { token = TEOF; break; }
+ { token = (start == EndOfFile()) ? TEOF : TNUL; break; }
yy4:
++p;
yy5:
@@ -229,20 +288,19 @@
{ token = NEWLINE; break; }
yy8:
yych = *++p;
- if (yych == '\n') goto yy28;
+ if (yych == '\n') goto yy6;
goto yy5;
yy9:
yyaccept = 0;
- q = ++p;
- yych = *p;
+ yych = *(q = ++p);
if (yybm[0+yych] & 32) {
goto yy9;
}
if (yych <= '\f') {
if (yych == '\n') goto yy6;
} else {
- if (yych <= '\r') goto yy30;
- if (yych == '#') goto yy32;
+ if (yych <= '\r') goto yy28;
+ if (yych == '#') goto yy30;
}
yy11:
{ token = INDENT; break; }
@@ -250,10 +308,9 @@
yyaccept = 1;
yych = *(q = ++p);
if (yych <= 0x00) goto yy5;
- goto yy33;
+ goto yy31;
yy13:
- ++p;
- yych = *p;
+ yych = *++p;
yy14:
if (yybm[0+yych] & 64) {
goto yy13;
@@ -267,93 +324,100 @@
{ token = EQUALS; break; }
yy20:
yych = *++p;
- if (yych == 'u') goto yy36;
+ if (yych == 'u') goto yy35;
goto yy14;
yy21:
yych = *++p;
- if (yych == 'e') goto yy37;
+ if (yych == 'e') goto yy36;
goto yy14;
yy22:
yych = *++p;
- if (yych == 'n') goto yy38;
+ if (yych == 'n') goto yy37;
goto yy14;
yy23:
yych = *++p;
- if (yych == 'o') goto yy39;
+ if (yych == 'o') goto yy38;
goto yy14;
yy24:
yych = *++p;
- if (yych == 'u') goto yy40;
+ if (yych == 'u') goto yy39;
goto yy14;
yy25:
yych = *++p;
- if (yych == 'u') goto yy41;
+ if (yych == 'u') goto yy40;
goto yy14;
yy26:
- ++p;
- if ((yych = *p) == '|') goto yy42;
+ yych = *++p;
+ if (yych == '@') goto yy41;
+ if (yych == '|') goto yy43;
{ token = PIPE; break; }
yy28:
- ++p;
- { token = NEWLINE; break; }
-yy30:
yych = *++p;
- if (yych == '\n') goto yy28;
-yy31:
+ if (yych == '\n') goto yy6;
+yy29:
p = q;
if (yyaccept == 0) {
goto yy11;
} else {
goto yy5;
}
+yy30:
+ yych = *++p;
+yy31:
+ if (yybm[0+yych] & 128) {
+ goto yy30;
+ }
+ if (yych <= 0x00) goto yy29;
+ if (yych >= '\v') {
+ r = p;
+ goto yy34;
+ }
+ r = p;
yy32:
++p;
- yych = *p;
-yy33:
- if (yybm[0+yych] & 128) {
- goto yy32;
- }
- if (yych <= 0x00) goto yy31;
- ++p;
+ p = r;
{ continue; }
+yy34:
+ yych = *++p;
+ if (yych == '\n') goto yy32;
+ goto yy29;
+yy35:
+ yych = *++p;
+ if (yych == 'i') goto yy45;
+ goto yy14;
yy36:
yych = *++p;
- if (yych == 'i') goto yy44;
+ if (yych == 'f') goto yy46;
goto yy14;
yy37:
yych = *++p;
- if (yych == 'f') goto yy45;
+ if (yych == 'c') goto yy47;
goto yy14;
yy38:
yych = *++p;
- if (yych == 'c') goto yy46;
+ if (yych == 'o') goto yy48;
goto yy14;
yy39:
yych = *++p;
- if (yych == 'o') goto yy47;
+ if (yych == 'l') goto yy49;
goto yy14;
yy40:
yych = *++p;
- if (yych == 'l') goto yy48;
+ if (yych == 'b') goto yy50;
goto yy14;
yy41:
- yych = *++p;
- if (yych == 'b') goto yy49;
- goto yy14;
-yy42:
+ ++p;
+ { token = PIPEAT; break; }
+yy43:
++p;
{ token = PIPE2; break; }
-yy44:
- yych = *++p;
- if (yych == 'l') goto yy50;
- goto yy14;
yy45:
yych = *++p;
- if (yych == 'a') goto yy51;
+ if (yych == 'l') goto yy51;
goto yy14;
yy46:
yych = *++p;
- if (yych == 'l') goto yy52;
+ if (yych == 'a') goto yy52;
goto yy14;
yy47:
yych = *++p;
@@ -361,87 +425,91 @@
goto yy14;
yy48:
yych = *++p;
- if (yych == 'e') goto yy55;
+ if (yych == 'l') goto yy54;
goto yy14;
yy49:
yych = *++p;
- if (yych == 'n') goto yy57;
+ if (yych == 'e') goto yy56;
goto yy14;
yy50:
yych = *++p;
- if (yych == 'd') goto yy58;
+ if (yych == 'n') goto yy58;
goto yy14;
yy51:
yych = *++p;
- if (yych == 'u') goto yy60;
+ if (yych == 'd') goto yy59;
goto yy14;
yy52:
yych = *++p;
if (yych == 'u') goto yy61;
goto yy14;
yy53:
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+ yych = *++p;
+ if (yych == 'u') goto yy62;
+ goto yy14;
+yy54:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = POOL; break; }
-yy55:
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+yy56:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = RULE; break; }
-yy57:
- yych = *++p;
- if (yych == 'i') goto yy62;
- goto yy14;
yy58:
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+ yych = *++p;
+ if (yych == 'i') goto yy63;
+ goto yy14;
+yy59:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = BUILD; break; }
-yy60:
- yych = *++p;
- if (yych == 'l') goto yy63;
- goto yy14;
yy61:
yych = *++p;
- if (yych == 'd') goto yy64;
+ if (yych == 'l') goto yy64;
goto yy14;
yy62:
yych = *++p;
- if (yych == 'n') goto yy65;
+ if (yych == 'd') goto yy65;
goto yy14;
yy63:
yych = *++p;
- if (yych == 't') goto yy66;
+ if (yych == 'n') goto yy66;
goto yy14;
yy64:
yych = *++p;
- if (yych == 'e') goto yy68;
+ if (yych == 't') goto yy67;
goto yy14;
yy65:
yych = *++p;
- if (yych == 'j') goto yy70;
+ if (yych == 'e') goto yy69;
goto yy14;
yy66:
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+ yych = *++p;
+ if (yych == 'j') goto yy71;
+ goto yy14;
+yy67:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = DEFAULT; break; }
-yy68:
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+yy69:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = INCLUDE; break; }
-yy70:
+yy71:
yych = *++p;
if (yych != 'a') goto yy14;
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = SUBNINJA; break; }
@@ -456,6 +524,119 @@
return token;
}
+bool Lexer::PeekIndent() {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ for (;;) {
+ start = p;
+
+{
+ unsigned char yych;
+ unsigned int yyaccept = 0;
+ static const unsigned char yybm[] = {
+ 0, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 0, 128, 128, 0, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 192, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ };
+ yych = *p;
+ if (yybm[0+yych] & 64) {
+ goto yy81;
+ }
+ if (yych <= '\f') {
+ if (yych == '\n') goto yy78;
+ } else {
+ if (yych <= '\r') goto yy80;
+ if (yych == '#') goto yy84;
+ }
+ ++p;
+yy77:
+ { last_token_ = ofs_ = start; return false; }
+yy78:
+ ++p;
+ { last_token_ = ofs_ = start; return false; }
+yy80:
+ yych = *++p;
+ if (yych == '\n') goto yy78;
+ goto yy77;
+yy81:
+ yyaccept = 0;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 64) {
+ goto yy81;
+ }
+ if (yych <= '\f') {
+ if (yych == '\n') goto yy78;
+ } else {
+ if (yych <= '\r') goto yy85;
+ if (yych == '#') goto yy87;
+ }
+yy83:
+ { last_token_ = start; ofs_ = p; return true; }
+yy84:
+ yyaccept = 1;
+ yych = *(q = ++p);
+ if (yych <= 0x00) goto yy77;
+ goto yy88;
+yy85:
+ yych = *++p;
+ if (yych == '\n') goto yy78;
+yy86:
+ p = q;
+ if (yyaccept == 0) {
+ goto yy83;
+ } else {
+ goto yy77;
+ }
+yy87:
+ yych = *++p;
+yy88:
+ if (yybm[0+yych] & 128) {
+ goto yy87;
+ }
+ if (yych <= 0x00) goto yy86;
+ if (yych >= '\v') goto yy91;
+yy89:
+ ++p;
+ { continue; }
+yy91:
+ yych = *++p;
+ if (yych == '\n') goto yy89;
+ goto yy86;
+}
+
+ }
+}
+
bool Lexer::PeekToken(Token token) {
Token t = ReadToken();
if (t == token)
@@ -508,47 +689,43 @@
};
yych = *p;
if (yybm[0+yych] & 128) {
- goto yy79;
+ goto yy98;
}
- if (yych <= 0x00) goto yy75;
- if (yych == '$') goto yy82;
- goto yy77;
-yy75:
+ if (yych <= 0x00) goto yy94;
+ if (yych == '$') goto yy101;
+ goto yy96;
+yy94:
++p;
{ break; }
-yy77:
+yy96:
++p;
-yy78:
+yy97:
{ break; }
-yy79:
- ++p;
- yych = *p;
- if (yybm[0+yych] & 128) {
- goto yy79;
- }
- { continue; }
-yy82:
- yych = *(q = ++p);
- if (yych == '\n') goto yy83;
- if (yych == '\r') goto yy85;
- goto yy78;
-yy83:
- ++p;
- { continue; }
-yy85:
+yy98:
yych = *++p;
- if (yych == '\n') goto yy87;
- p = q;
- goto yy78;
-yy87:
+ if (yybm[0+yych] & 128) {
+ goto yy98;
+ }
+ { continue; }
+yy101:
+ yych = *(q = ++p);
+ if (yych == '\n') goto yy102;
+ if (yych == '\r') goto yy104;
+ goto yy97;
+yy102:
++p;
{ continue; }
+yy104:
+ yych = *++p;
+ if (yych == '\n') goto yy102;
+ p = q;
+ goto yy97;
}
}
}
-bool Lexer::ReadIdent(string* out) {
+bool Lexer::ReadIdent(StringPiece* out) {
const char* p = ofs_;
const char* start;
for (;;) {
@@ -592,21 +769,20 @@
};
yych = *p;
if (yybm[0+yych] & 128) {
- goto yy93;
+ goto yy110;
}
++p;
{
last_token_ = start;
return false;
}
-yy93:
- ++p;
- yych = *p;
+yy110:
+ yych = *++p;
if (yybm[0+yych] & 128) {
- goto yy93;
+ goto yy110;
}
{
- out->assign(start, p - start);
+ *out = StringPiece(start, p - start);
break;
}
}
@@ -618,7 +794,7 @@
return true;
}
-bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) {
+bool Lexer::ReadBindingValue(StringPiece* out, string* err) {
const char* p = ofs_;
const char* q;
const char* start;
@@ -627,6 +803,1126 @@
{
unsigned char yych;
+ unsigned int yyaccept = 0;
+ static const unsigned char yybm[] = {
+ 0, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 0, 64, 64, 0, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 0, 64, 64, 64,
+ 64, 64, 64, 64, 64, 192, 192, 64,
+ 192, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 64, 64, 64, 64, 64, 64,
+ 64, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 64, 64, 64, 64, 192,
+ 64, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ };
+ yych = *p;
+ if (yybm[0+yych] & 64) {
+ goto yy117;
+ }
+ if (yych <= 0x00) goto yy115;
+ if (yych <= '\n') goto yy120;
+ if (yych <= '\r') goto yy122;
+ goto yy124;
+yy115:
+ ++p;
+ {
+ last_token_ = start;
+ return UnexpectedNulError(start, err);
+ }
+yy117:
+ yyaccept = 0;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 64) {
+ goto yy117;
+ }
+ if (yych >= 0x0E) goto yy125;
+yy119:
+ {
+ continue;
+ }
+yy120:
+ ++p;
+ {
+ break;
+ }
+yy122:
+ yych = *++p;
+ if (yych == '\n') goto yy120;
+ {
+ last_token_ = start;
+ return Error(DescribeLastError(), err);
+ }
+yy124:
+ yych = *++p;
+ if (yych <= '-') {
+ if (yych <= 0x1F) {
+ if (yych <= '\n') {
+ if (yych <= '\t') goto yy127;
+ goto yy117;
+ } else {
+ if (yych == '\r') goto yy129;
+ goto yy127;
+ }
+ } else {
+ if (yych <= '#') {
+ if (yych <= ' ') goto yy117;
+ goto yy127;
+ } else {
+ if (yych <= '$') goto yy117;
+ if (yych <= ',') goto yy127;
+ goto yy117;
+ }
+ }
+ } else {
+ if (yych <= '^') {
+ if (yych <= ':') {
+ if (yych <= '/') goto yy127;
+ goto yy117;
+ } else {
+ if (yych <= '@') goto yy127;
+ if (yych <= 'Z') goto yy117;
+ goto yy127;
+ }
+ } else {
+ if (yych <= '`') {
+ if (yych <= '_') goto yy117;
+ goto yy127;
+ } else {
+ if (yych <= 'z') goto yy117;
+ if (yych <= '{') goto yy130;
+ goto yy127;
+ }
+ }
+ }
+yy125:
+ yych = *++p;
+ if (yych <= '-') {
+ if (yych <= 0x1F) {
+ if (yych <= '\n') {
+ if (yych >= '\n') goto yy117;
+ } else {
+ if (yych == '\r') goto yy131;
+ }
+ } else {
+ if (yych <= '#') {
+ if (yych <= ' ') goto yy117;
+ } else {
+ if (yych <= '$') goto yy117;
+ if (yych >= '-') goto yy117;
+ }
+ }
+ } else {
+ if (yych <= '^') {
+ if (yych <= ':') {
+ if (yych >= '0') goto yy117;
+ } else {
+ if (yych <= '@') goto yy126;
+ if (yych <= 'Z') goto yy117;
+ }
+ } else {
+ if (yych <= '`') {
+ if (yych <= '_') goto yy117;
+ } else {
+ if (yych <= 'z') goto yy117;
+ if (yych <= '{') goto yy132;
+ }
+ }
+ }
+yy126:
+ p = q;
+ if (yyaccept == 0) {
+ goto yy119;
+ } else {
+ goto yy128;
+ }
+yy127:
+ ++p;
+yy128:
+ {
+ last_token_ = start;
+ return Error("bad $-escape (literal $ must be written as $$)", err);
+ }
+yy129:
+ yych = *++p;
+ if (yych == '\n') goto yy117;
+ goto yy128;
+yy130:
+ yyaccept = 1;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 128) {
+ goto yy133;
+ }
+ goto yy128;
+yy131:
+ yych = *++p;
+ if (yych == '\n') goto yy117;
+ goto yy126;
+yy132:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy133;
+ }
+ goto yy126;
+yy133:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy133;
+ }
+ if (yych == '}') goto yy117;
+ goto yy126;
+}
+
+ }
+ *out = StringPiece(ofs_, p - ofs_);
+ last_token_ = start;
+ ofs_ = p;
+ // Non-path strings end in newlines, so there's no whitespace to eat.
+ return true;
+}
+
+StringPiece Lexer::PeekCanonicalPath() {
+ auto finish = [this](const char* start, const char* end) {
+ ofs_ = end;
+ EatWhitespace();
+ return StringPiece(start, end - start);
+ };
+
+ const char* p = ofs_;
+ const char* q;
+ const char* r;
+ last_token_ = ofs_;
+
+ do {
+
+{
+ unsigned char yych;
+ static const unsigned char yybm[] = {
+ 0, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 0, 128, 128, 0, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 0, 128, 128, 128, 0, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 0,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 0, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 0, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ };
+ yych = *p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy137;
+ if (yych != '\n') goto yy139;
+ } else {
+ if (yych <= '\r') goto yy137;
+ if (yych != ' ') goto yy139;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy137;
+ if (yych <= '-') goto yy139;
+ if (yych <= '.') goto yy140;
+ goto yy141;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy139;
+ } else {
+ if (yych != '|') goto yy139;
+ }
+ }
+ }
+yy137:
+ ++p;
+yy138:
+ { break; }
+yy139:
+ yych = *(q = ++p);
+ if (yych <= 0x00) goto yy138;
+ if (yych == '$') goto yy138;
+ goto yy143;
+yy140:
+ yych = *(q = ++p);
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy138;
+ if (yych == '\n') goto yy138;
+ goto yy142;
+ } else {
+ if (yych <= '\r') goto yy138;
+ if (yych == ' ') goto yy138;
+ goto yy142;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy138;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy149;
+ goto yy150;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy142;
+ goto yy138;
+ } else {
+ if (yych == '|') goto yy138;
+ goto yy142;
+ }
+ }
+ }
+yy141:
+ yych = *(q = ++p);
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy138;
+ if (yych == '\n') goto yy138;
+ } else {
+ if (yych <= '\r') goto yy138;
+ if (yych == ' ') goto yy138;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy138;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy151;
+ goto yy138;
+ } else {
+ if (yych <= ':') {
+ if (yych >= ':') goto yy138;
+ } else {
+ if (yych == '|') goto yy138;
+ }
+ }
+ }
+yy142:
+ yych = *++p;
+yy143:
+ if (yybm[0+yych] & 128) {
+ goto yy142;
+ }
+ if (yych <= '\r') {
+ if (yych <= 0x00) goto yy144;
+ if (yych <= '\n') {
+ r = p;
+ goto yy145;
+ }
+ r = p;
+ goto yy147;
+ } else {
+ if (yych <= ' ') {
+ r = p;
+ goto yy145;
+ }
+ if (yych <= '$') goto yy144;
+ if (yych <= '/') goto yy148;
+ r = p;
+ goto yy145;
+ }
+yy144:
+ p = q;
+ goto yy138;
+yy145:
+ ++p;
+ p = r;
+ { return finish(ofs_, p); }
+yy147:
+ yych = *++p;
+ if (yych == '\n') goto yy145;
+ goto yy144;
+yy148:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy142;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy142;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy151;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy142;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy142;
+ }
+ }
+ }
+yy149:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy142;
+ }
+ if (yych <= '$') goto yy144;
+ if (yych <= '/') goto yy152;
+ goto yy144;
+yy150:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy153;
+ if (yych <= '.') goto yy155;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy153;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+ }
+yy151:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy142;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy142;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy156;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy142;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy142;
+ }
+ }
+ }
+yy152:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy142;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy142;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy157;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy142;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy142;
+ }
+ }
+ }
+yy153:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') {
+ r = p;
+ goto yy158;
+ }
+ goto yy153;
+ } else {
+ if (yych <= '\r') {
+ r = p;
+ goto yy160;
+ }
+ if (yych == ' ') {
+ r = p;
+ goto yy158;
+ }
+ goto yy153;
+ }
+ } else {
+ if (yych <= '9') {
+ if (yych <= '$') goto yy144;
+ if (yych == '/') goto yy161;
+ goto yy153;
+ } else {
+ if (yych <= ':') {
+ r = p;
+ goto yy158;
+ }
+ if (yych == '|') {
+ r = p;
+ goto yy158;
+ }
+ goto yy153;
+ }
+ }
+yy155:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy153;
+ if (yych <= '.') goto yy162;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy153;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+ }
+yy156:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy142;
+ }
+ goto yy144;
+yy157:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy142;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy142;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy149;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy142;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy142;
+ }
+ }
+ }
+yy158:
+ ++p;
+ p = r;
+ { return finish(ofs_ + 2, p); }
+yy160:
+ yych = *++p;
+ if (yych == '\n') goto yy158;
+ goto yy144;
+yy161:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy153;
+ if (yych <= '.') goto yy163;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy153;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+ }
+yy162:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '9') {
+ if (yych <= '$') goto yy144;
+ if (yych == '/') goto yy150;
+ goto yy153;
+ } else {
+ if (yych <= ':') goto yy144;
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+yy163:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy153;
+ if (yych >= '/') goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy153;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+ }
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '9') {
+ if (yych <= '$') goto yy144;
+ if (yych == '/') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= ':') goto yy144;
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+}
+
+ } while (false);
+
+ return {};
+}
+
+bool Lexer::ReadPath(LexedPath* out, std::string* err) {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ for (;;) {
+ start = p;
+
+{
+ unsigned char yych;
+ unsigned int yyaccept = 0;
+ static const unsigned char yybm[] = {
+ 0, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 0, 32, 32, 0, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 64, 32, 32, 32, 0, 32, 32, 32,
+ 32, 32, 32, 32, 32, 160, 160, 32,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 0, 32, 32, 32, 32, 32,
+ 32, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 32, 32, 32, 32, 160,
+ 32, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 32, 0, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ };
+ yych = *p;
+ if (yybm[0+yych] & 32) {
+ goto yy169;
+ }
+ if (yych <= '\r') {
+ if (yych <= 0x00) goto yy167;
+ if (yych <= '\n') goto yy172;
+ goto yy174;
+ } else {
+ if (yych <= ' ') goto yy172;
+ if (yych <= '$') goto yy176;
+ goto yy172;
+ }
+yy167:
+ ++p;
+ {
+ last_token_ = start;
+ return UnexpectedNulError(start, err);
+ }
+yy169:
+ yyaccept = 0;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 32) {
+ goto yy169;
+ }
+ if (yych <= ' ') goto yy171;
+ if (yych <= '$') goto yy177;
+yy171:
+ {
+ continue;
+ }
+yy172:
+ ++p;
+ {
+ p = start;
+ break;
+ }
+yy174:
+ yych = *++p;
+ if (yych == '\n') goto yy172;
+ {
+ last_token_ = start;
+ return Error(DescribeLastError(), err);
+ }
+yy176:
+ yych = *++p;
+ if (yych <= '-') {
+ if (yych <= 0x1F) {
+ if (yych <= '\n') {
+ if (yych <= '\t') goto yy179;
+ goto yy181;
+ } else {
+ if (yych == '\r') goto yy183;
+ goto yy179;
+ }
+ } else {
+ if (yych <= '#') {
+ if (yych <= ' ') goto yy169;
+ goto yy179;
+ } else {
+ if (yych <= '$') goto yy169;
+ if (yych <= ',') goto yy179;
+ goto yy169;
+ }
+ }
+ } else {
+ if (yych <= '^') {
+ if (yych <= ':') {
+ if (yych <= '/') goto yy179;
+ goto yy169;
+ } else {
+ if (yych <= '@') goto yy179;
+ if (yych <= 'Z') goto yy169;
+ goto yy179;
+ }
+ } else {
+ if (yych <= '`') {
+ if (yych <= '_') goto yy169;
+ goto yy179;
+ } else {
+ if (yych <= 'z') goto yy169;
+ if (yych <= '{') goto yy184;
+ goto yy179;
+ }
+ }
+ }
+yy177:
+ yych = *++p;
+ if (yych <= '-') {
+ if (yych <= 0x1F) {
+ if (yych <= '\n') {
+ if (yych >= '\n') goto yy181;
+ } else {
+ if (yych == '\r') goto yy185;
+ }
+ } else {
+ if (yych <= '#') {
+ if (yych <= ' ') goto yy169;
+ } else {
+ if (yych <= '$') goto yy169;
+ if (yych >= '-') goto yy169;
+ }
+ }
+ } else {
+ if (yych <= '^') {
+ if (yych <= ':') {
+ if (yych >= '0') goto yy169;
+ } else {
+ if (yych <= '@') goto yy178;
+ if (yych <= 'Z') goto yy169;
+ }
+ } else {
+ if (yych <= '`') {
+ if (yych <= '_') goto yy169;
+ } else {
+ if (yych <= 'z') goto yy169;
+ if (yych <= '{') goto yy186;
+ }
+ }
+ }
+yy178:
+ p = q;
+ if (yyaccept == 0) {
+ goto yy171;
+ } else {
+ goto yy180;
+ }
+yy179:
+ ++p;
+yy180:
+ {
+ last_token_ = start;
+ return Error("bad $-escape (literal $ must be written as $$)", err);
+ }
+yy181:
+ yyaccept = 0;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 32) {
+ goto yy169;
+ }
+ if (yych <= '\r') goto yy171;
+ if (yych <= ' ') goto yy181;
+ if (yych <= '$') goto yy177;
+ goto yy171;
+yy183:
+ yych = *++p;
+ if (yych == '\n') goto yy181;
+ goto yy180;
+yy184:
+ yyaccept = 1;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 128) {
+ goto yy187;
+ }
+ goto yy180;
+yy185:
+ yych = *++p;
+ if (yych == '\n') goto yy181;
+ goto yy178;
+yy186:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy187;
+ }
+ goto yy178;
+yy187:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy187;
+ }
+ if (yych == '}') goto yy169;
+ goto yy178;
+}
+
+ }
+ *out = {};
+ out->str_ = StringPiece(ofs_, p - ofs_);
+ last_token_ = start;
+ ofs_ = p;
+ EatWhitespace();
+ return true;
+}
+
+/// Append the let binding's evaluated value to the output string. The input
+/// StringPiece must include a valid binding terminator.
+template <typename EvalVar>
+static inline void EvaluateBinding(std::string* out_append, StringPiece value,
+ EvalVar&& eval_var) {
+ auto expand = [&eval_var](const char* start, const char* end) {
+ StringPiece var(start, end - start);
+ eval_var(var);
+ };
+
+ const char* p = value.data();
+ const char* q;
+
+ for (;;) {
+ const char* start = p;
+
+{
+ unsigned char yych;
+ static const unsigned char yybm[] = {
+ 0, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 0, 16, 16, 0, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 48, 16, 16, 16, 0, 16, 16, 16,
+ 16, 16, 16, 16, 16, 208, 144, 16,
+ 208, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 16, 16, 16, 16, 16, 16,
+ 16, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 16, 16, 16, 16, 208,
+ 16, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ };
+ yych = *p;
+ if (yybm[0+yych] & 16) {
+ goto yy193;
+ }
+ if (yych <= 0x00) goto yy191;
+ if (yych <= '\n') goto yy196;
+ if (yych <= '\r') goto yy198;
+ goto yy199;
+yy191:
+ ++p;
+yy192:
+ { assert(false && "bad input in EvaluateBinding"); abort(); }
+yy193:
+ yych = *++p;
+ if (yybm[0+yych] & 16) {
+ goto yy193;
+ }
+ { out_append->append(start, p - start); continue; }
+yy196:
+ ++p;
+ { break; }
+yy198:
+ yych = *++p;
+ if (yych == '\n') goto yy196;
+ goto yy192;
+yy199:
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 64) {
+ goto yy207;
+ }
+ if (yych <= ' ') {
+ if (yych <= '\f') {
+ if (yych != '\n') goto yy192;
+ } else {
+ if (yych <= '\r') goto yy203;
+ if (yych <= 0x1F) goto yy192;
+ goto yy205;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych == '$') goto yy205;
+ goto yy192;
+ } else {
+ if (yych <= ':') goto yy205;
+ if (yych <= '`') goto yy192;
+ if (yych <= '{') goto yy210;
+ goto yy192;
+ }
+ }
+yy200:
+ yych = *++p;
+ if (yybm[0+yych] & 32) {
+ goto yy200;
+ }
+ { continue; }
+yy203:
+ yych = *++p;
+ if (yych == '\n') goto yy200;
+yy204:
+ p = q;
+ goto yy192;
+yy205:
+ ++p;
+ { out_append->push_back(start[1]); continue; }
+yy207:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
+ goto yy207;
+ }
+ { expand(start + 1, p); continue; }
+yy210:
+ yych = *++p;
+ if (yych == '}') goto yy204;
+ goto yy212;
+yy211:
+ yych = *++p;
+yy212:
+ if (yybm[0+yych] & 128) {
+ goto yy211;
+ }
+ if (yych != '}') goto yy204;
+ ++p;
+ { expand(start + 2, p - 1); continue; }
+}
+
+ }
+ assert((p == value.data() + value.size()) &&
+ "bad end pos in EvaluateBinding");
+}
+
+void EvaluateBindingInScope(std::string* out_append, StringPiece value,
+ ScopePosition pos) {
+ EvaluateBinding(out_append, value,
+ [out_append, &pos](const HashedStrView& var) {
+ Scope::EvaluateVariableAtPos(out_append, var, pos);
+ });
+}
+
+bool EvaluateBindingOnRule(std::string* out_append, StringPiece value,
+ EdgeEval* edge_eval, std::string* err) {
+ bool result = true;
+ EvaluateBinding(out_append, value,
+ [out_append, edge_eval, &result, err](const HashedStrView& var) {
+ result = result && edge_eval->EvaluateVariable(out_append, var, err);
+ });
+ return result;
+}
+
+std::string EvaluateBindingForTesting(StringPiece value) {
+ std::string result;
+ EvaluateBinding(&result, value, [&result](StringPiece var) {
+ result += "[$" + var.AsString() + "]";
+ });
+ return result;
+}
+
+/// Append an evaluated path to the output string.
+///
+/// This function does not canonicalize the output. Ninja canonicalizes paths for
+/// build nodes, but not all paths (e.g. It doesn't canonicalize paths to
+/// included ninja files.)
+template <typename EvalVar>
+static inline void EvaluatePath(std::string* out_append, const LexedPath& path,
+ EvalVar&& eval_var) {
+ auto expand = [&eval_var](const char* start, const char* end) {
+ StringPiece var(start, end - start);
+ eval_var(var);
+ };
+
+ const char* p = path.str_.data();
+ const char* q;
+
+ for (;;) {
+ const char* start = p;
+
+{
+ unsigned char yych;
static const unsigned char yybm[] = {
0, 16, 16, 16, 16, 16, 16, 16,
16, 16, 0, 16, 16, 0, 16, 16,
@@ -663,168 +1959,121 @@
};
yych = *p;
if (yybm[0+yych] & 16) {
- goto yy100;
+ goto yy219;
}
if (yych <= '\r') {
- if (yych <= 0x00) goto yy98;
- if (yych <= '\n') goto yy103;
- goto yy105;
+ if (yych <= 0x00) goto yy217;
+ if (yych <= '\n') goto yy222;
+ goto yy224;
} else {
- if (yych <= ' ') goto yy103;
- if (yych <= '$') goto yy107;
- goto yy103;
+ if (yych <= ' ') goto yy222;
+ if (yych <= '$') goto yy225;
+ goto yy222;
}
-yy98:
+yy217:
++p;
- {
- last_token_ = start;
- return Error("unexpected EOF", err);
- }
-yy100:
- ++p;
- yych = *p;
- if (yybm[0+yych] & 16) {
- goto yy100;
- }
- {
- eval->AddText(StringPiece(start, p - start));
- continue;
- }
-yy103:
- ++p;
- {
- if (path) {
- p = start;
- break;
- } else {
- if (*start == '\n')
- break;
- eval->AddText(StringPiece(start, 1));
- continue;
- }
- }
-yy105:
- ++p;
- if ((yych = *p) == '\n') goto yy108;
- {
- last_token_ = start;
- return Error(DescribeLastError(), err);
- }
-yy107:
+yy218:
+ { assert(false && "bad input in EvaluatePath"); abort(); }
+yy219:
yych = *++p;
+ if (yybm[0+yych] & 16) {
+ goto yy219;
+ }
+ { out_append->append(start, p - start); continue; }
+yy222:
+ ++p;
+ { p = start; break; }
+yy224:
+ yych = *++p;
+ if (yych == '\n') goto yy222;
+ goto yy218;
+yy225:
+ yych = *(q = ++p);
if (yybm[0+yych] & 64) {
- goto yy120;
+ goto yy233;
}
if (yych <= ' ') {
if (yych <= '\f') {
- if (yych == '\n') goto yy112;
- goto yy110;
+ if (yych != '\n') goto yy218;
} else {
- if (yych <= '\r') goto yy115;
- if (yych <= 0x1F) goto yy110;
- goto yy116;
+ if (yych <= '\r') goto yy229;
+ if (yych <= 0x1F) goto yy218;
+ goto yy231;
}
} else {
if (yych <= '/') {
- if (yych == '$') goto yy118;
- goto yy110;
+ if (yych == '$') goto yy231;
+ goto yy218;
} else {
- if (yych <= ':') goto yy123;
- if (yych <= '`') goto yy110;
- if (yych <= '{') goto yy125;
- goto yy110;
+ if (yych <= ':') goto yy231;
+ if (yych <= '`') goto yy218;
+ if (yych <= '{') goto yy236;
+ goto yy218;
}
}
-yy108:
- ++p;
- {
- if (path)
- p = start;
- break;
- }
-yy110:
- ++p;
-yy111:
- {
- last_token_ = start;
- return Error("bad $-escape (literal $ must be written as $$)", err);
- }
-yy112:
- ++p;
- yych = *p;
- if (yybm[0+yych] & 32) {
- goto yy112;
- }
- {
- continue;
- }
-yy115:
+yy226:
yych = *++p;
- if (yych == '\n') goto yy126;
- goto yy111;
-yy116:
- ++p;
- {
- eval->AddText(StringPiece(" ", 1));
- continue;
- }
-yy118:
- ++p;
- {
- eval->AddText(StringPiece("$", 1));
- continue;
- }
-yy120:
- ++p;
- yych = *p;
- if (yybm[0+yych] & 64) {
- goto yy120;
+ if (yybm[0+yych] & 32) {
+ goto yy226;
}
- {
- eval->AddSpecial(StringPiece(start + 1, p - start - 1));
- continue;
- }
-yy123:
- ++p;
- {
- eval->AddText(StringPiece(":", 1));
- continue;
- }
-yy125:
- yych = *(q = ++p);
- if (yybm[0+yych] & 128) {
- goto yy129;
- }
- goto yy111;
-yy126:
- ++p;
- yych = *p;
- if (yych == ' ') goto yy126;
- {
- continue;
- }
-yy129:
- ++p;
- yych = *p;
- if (yybm[0+yych] & 128) {
- goto yy129;
- }
- if (yych == '}') goto yy132;
+ { continue; }
+yy229:
+ yych = *++p;
+ if (yych == '\n') goto yy226;
+yy230:
p = q;
- goto yy111;
-yy132:
+ goto yy218;
+yy231:
++p;
- {
- eval->AddSpecial(StringPiece(start + 2, p - start - 3));
- continue;
- }
+ { out_append->push_back(start[1]); continue; }
+yy233:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
+ goto yy233;
+ }
+ { expand(start + 1, p); continue; }
+yy236:
+ yych = *++p;
+ if (yych == '}') goto yy230;
+ goto yy238;
+yy237:
+ yych = *++p;
+yy238:
+ if (yybm[0+yych] & 128) {
+ goto yy237;
+ }
+ if (yych != '}') goto yy230;
+ ++p;
+ { expand(start + 2, p - 1); continue; }
}
}
- last_token_ = start;
- ofs_ = p;
- if (path)
- EatWhitespace();
- // Non-path strings end in newlines, so there's no whitespace to eat.
- return true;
+ assert((p == path.str_.data() + path.str_.size()) &&
+ "bad end pos in EvaluatePath");
+}
+
+void EvaluatePathInScope(std::string* out_append, const LexedPath& path,
+ ScopePosition pos) {
+ EvaluatePath(out_append, path, [out_append, &pos](const HashedStrView& var) {
+ Scope::EvaluateVariableAtPos(out_append, var, pos);
+ });
+}
+
+void EvaluatePathOnEdge(std::string* out_append, const LexedPath& path,
+ const Edge& edge) {
+ EvaluatePath(out_append, path, [out_append, &edge](const HashedStrView& var) {
+ // First look for a binding on the edge itself, then fall back to the
+ // edge's enclosing scope.
+ if (edge.EvaluateVariableSelfOnly(out_append, var))
+ return;
+ Scope::EvaluateVariableAtPos(out_append, var, edge.pos_.scope_pos());
+ });
+}
+
+std::string EvaluatePathForTesting(const LexedPath& path) {
+ std::string result;
+ EvaluatePath(&result, path, [&result](StringPiece var) {
+ result += "[$" + var.AsString() + "]";
+ });
+ return result;
}
diff --git a/src/lexer.h b/src/lexer.h
index f366556..42581a7 100644
--- a/src/lexer.h
+++ b/src/lexer.h
@@ -15,19 +15,46 @@
#ifndef NINJA_LEXER_H_
#define NINJA_LEXER_H_
+#include <assert.h>
+
+#include <string>
+
#include "string_piece.h"
+#include "util.h"
// Windows may #define ERROR.
#ifdef ERROR
#undef ERROR
#endif
+struct Edge;
+struct EdgeEval;
struct EvalString;
+struct LexedPath;
+struct ScopePosition;
+
+/// Search the manifest for the next acceptable point to split the manifest.
+/// Assuming the manifest is syntactically valid to the returned position, the
+/// chunk parser will finish parsing a declaration just before the returned
+/// position.
+size_t AdvanceToNextManifestChunk(StringPiece content, size_t idx);
+
+bool DecorateErrorWithLocation(const std::string& filename,
+ const char* file_start,
+ size_t file_offset,
+ const std::string& message,
+ std::string* err);
struct Lexer {
- Lexer() {}
+ /// Ordinary lexer constructor.
+ Lexer(const std::string& filename, StringPiece file_content, const char* pos)
+ : filename_(filename), input_(file_content), ofs_(pos) {
+ assert(pos >= &file_content[0] &&
+ pos <= file_content.data() + file_content.size());
+ }
+
/// Helper ctor useful for tests.
- explicit Lexer(const char* input);
+ explicit Lexer(const char* input) : Lexer("input", input, input) {}
enum Token {
ERROR,
@@ -41,9 +68,11 @@
NEWLINE,
PIPE,
PIPE2,
+ PIPEAT,
POOL,
RULE,
SUBNINJA,
+ TNUL,
TEOF,
};
@@ -55,10 +84,17 @@
/// If the last token read was an ERROR token, provide more info
/// or the empty string.
- string DescribeLastError();
+ std::string DescribeLastError();
- /// Start parsing some input.
- void Start(StringPiece filename, StringPiece input);
+ // XXX: Decide whether to use char* or size_t to represent lexer position.
+ // (We're using size_t for diagnostic reporting -- GetLastTokenOffset, but
+ // we're using char* for checking for the end-of-chunk.)
+ const char* GetPos() { return ofs_; }
+ void ResetPos(const char* pos) { ofs_ = pos; last_token_ = nullptr; }
+
+ size_t GetLastTokenOffset() {
+ return last_token_ ? last_token_ - input_.data() : 0;
+ }
/// Read a Token from the Token enum.
Token ReadToken();
@@ -66,40 +102,86 @@
/// Rewind to the last read Token.
void UnreadToken();
+ /// If the next token is a binding's indentation, this function reads it and
+ /// returns true. This function skips lines containing only a comment, which
+ /// are allowed in a block of indented bindings. (A completely blank line, on
+ /// the other hand, ends the block.)
+ bool PeekIndent();
+
/// If the next token is \a token, read it and return true.
bool PeekToken(Token token);
/// Read a simple identifier (a rule or variable name).
/// Returns false if a name can't be read.
- bool ReadIdent(string* out);
+ bool ReadIdent(StringPiece* out);
- /// Read a path (complete with $escapes).
- /// Returns false only on error, returned path may be empty if a delimiter
- /// (space, newline) is hit.
- bool ReadPath(EvalString* path, string* err) {
- return ReadEvalString(path, true, err);
- }
+ /// Parse the value in a "var = value" let binding and return the unprocessed
+ /// span of the value (escapes and variable expansions are left as-is). The
+ /// binding must end with a newline, which is consumed.
+ ///
+ /// The returned StringPiece includes the EOL terminator, "\r\n" or "\n".
+ /// Copying the StringPiece copies the terminator.
+ bool ReadBindingValue(StringPiece* out, std::string* err);
- /// Read the value side of a var = value line (complete with $escapes).
- /// Returns false only on error.
- bool ReadVarValue(EvalString* value, string* err) {
- return ReadEvalString(value, false, err);
- }
+ /// Try to parse a path from the manifest that's already canonical and doesn't
+ /// need evaluation. Returns the string if successful and return a 0-size
+ /// string otherwise. This function is a performance optimization.
+ StringPiece PeekCanonicalPath();
+
+ /// Read a ninja path (e.g. build/include declarations) and output a LexedPath
+ /// object with the position of the path in the loaded file's buffer. If there
+ /// is no path before the lexer sees a delimiter (e.g. space, newline, colon),
+ /// it returns true and outputs an empty string.
+ ///
+ /// On error, the function returns false and writes an error to *err.
+ ///
+ /// The delimiter is not included in the path's StringPiece, and (except for
+ /// space characters), the delimiter is left in the lexer's input stream.
+ ///
+ /// Because path evaluation requires the delimiter's presence, the path should
+ /// be evaluated before its underlying memory is freed, and the StringPiece
+ /// should not be duplicated. (e.g. Don't copy it into an std::string first.)
+ bool ReadPath(LexedPath* out, std::string* err);
/// Construct an error message with context.
- bool Error(const string& message, string* err);
+ bool Error(const std::string& message, std::string* err);
private:
+ /// Return the end of the file input (which points at a NUL byte).
+ const char* EndOfFile() { return input_.data() + input_.size(); }
+
+ /// Report an error on a NUL character, which could indicate EOF, but could
+ /// also be a stray NUL character in the file's content.
+ bool UnexpectedNulError(const char* pos, std::string* err);
+
/// Skip past whitespace (called after each read token/ident/etc.).
void EatWhitespace();
- /// Read a $-escaped string.
- bool ReadEvalString(EvalString* eval, bool path, string* err);
-
- StringPiece filename_;
+ std::string filename_;
StringPiece input_;
- const char* ofs_;
- const char* last_token_;
+ const char* ofs_ = nullptr;
+ const char* last_token_ = nullptr;
};
+void EvaluateBindingInScope(std::string* out_append,
+ StringPiece value,
+ ScopePosition pos);
+
+bool EvaluateBindingOnRule(std::string* out_append,
+ StringPiece value,
+ EdgeEval* edge_eval,
+ std::string* err);
+
+/// Used for lexer tests.
+std::string EvaluateBindingForTesting(StringPiece value);
+
+void EvaluatePathInScope(std::string* out_append, const LexedPath& path,
+ ScopePosition pos);
+
+void EvaluatePathOnEdge(std::string* out_append, const LexedPath& path,
+ const Edge& edge);
+
+/// Used for lexer tests.
+std::string EvaluatePathForTesting(const LexedPath& path);
+
#endif // NINJA_LEXER_H_
diff --git a/src/lexer.in.cc b/src/lexer.in.cc
index c1fb822..63fa937 100644
--- a/src/lexer.in.cc
+++ b/src/lexer.in.cc
@@ -17,24 +17,79 @@
#include <stdio.h>
#include "eval_env.h"
+#include "graph.h"
#include "util.h"
-bool Lexer::Error(const string& message, string* err) {
+size_t AdvanceToNextManifestChunk(StringPiece content, size_t idx) {
+ assert(idx <= content.size());
+
+ // Iterate over each LF in the manifest, starting at the given index.
+ while (true) {
+ const void* next_line = memchr(content.data() + idx, '\n',
+ content.size() - idx);
+ if (next_line == nullptr) {
+ break;
+ }
+ idx = static_cast<const char*>(next_line) - content.data();
+ ++idx; // step over the LF
+
+ // The line must not be preceded by a line continuator. This logic can
+ // filter out more split candidates than strictly necessary:
+ // - The preceding line could have a comment that ends with a "$": "# $\n"
+ // - The preceding line could end with an escaped-dollar: "X=$$\n"
+ if ((idx >= 2 && content.substr(idx - 2, 2) == "$\n") ||
+ (idx >= 3 && content.substr(idx - 3, 3) == "$\r\n")) {
+ continue;
+ }
+
+ // Skip an indented line or a comment line, either of which could be part of
+ // an earlier declaration. Ninja allows unindented comments (as well as
+ // indented comments) inside a binding block, e.g.:
+ //
+ // build foo: cc
+ // # comment-line
+ // pool = link_pool
+ //
+ // Ninja doesn't allow blank lines in a binding block. This code could
+ // probably allow a chunk to start with a blank line, but it seems better if
+ // it doesn't.
+ if (idx >= content.size() ||
+ content[idx] == ' ' || content[idx] == '#' ||
+ content[idx] == '\r' || content[idx] == '\n') {
+ continue;
+ }
+
+ return idx;
+ }
+
+ return content.size();
+}
+
+bool DecorateErrorWithLocation(const std::string& filename,
+ const char* file_start,
+ size_t file_offset,
+ const std::string& message,
+ std::string* err) {
+ // Make a copy in case message and err alias.
+ std::string message_tmp = message;
+
// Compute line/column.
int line = 1;
- const char* line_start = input_.str_;
- for (const char* p = input_.str_; p < last_token_; ++p) {
+ const char* line_start = file_start;
+ const char* file_pos = file_start + file_offset;
+
+ for (const char* p = line_start; p < file_pos; ++p) {
if (*p == '\n') {
++line;
line_start = p + 1;
}
}
- int col = last_token_ ? (int)(last_token_ - line_start) : 0;
+ int col = (int)(file_pos - line_start);
char buf[1024];
- snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line);
+ snprintf(buf, sizeof(buf), "%s:%d: ", filename.c_str(), line);
*err = buf;
- *err += message + "\n";
+ *err += message_tmp + "\n";
// Add some context to the message.
const int kTruncateColumn = 72;
@@ -58,15 +113,16 @@
return false;
}
-Lexer::Lexer(const char* input) {
- Start("input", input);
+bool Lexer::Error(const std::string& message, std::string* err) {
+ return DecorateErrorWithLocation(filename_, input_.data(),
+ GetLastTokenOffset(), message, err);
}
-void Lexer::Start(StringPiece filename, StringPiece input) {
- filename_ = filename;
- input_ = input;
- ofs_ = input_.str_;
- last_token_ = NULL;
+bool Lexer::UnexpectedNulError(const char* pos, std::string* err) {
+ assert(*pos == '\0');
+ const char* msg = (pos == EndOfFile()) ? "unexpected EOF"
+ : "unexpected NUL byte";
+ return Error(msg, err);
}
const char* Lexer::TokenName(Token t) {
@@ -82,9 +138,11 @@
case NEWLINE: return "newline";
case PIPE2: return "'||'";
case PIPE: return "'|'";
+ case PIPEAT: return "'|@'";
case POOL: return "'pool'";
case RULE: return "'rule'";
case SUBNINJA: return "'subninja'";
+ case TNUL: return "nul byte";
case TEOF: return "eof";
}
return NULL; // not reached
@@ -116,6 +174,7 @@
Lexer::Token Lexer::ReadToken() {
const char* p = ofs_;
const char* q;
+ const char* r;
const char* start;
Lexer::Token token;
for (;;) {
@@ -124,15 +183,17 @@
re2c:define:YYCTYPE = "unsigned char";
re2c:define:YYCURSOR = p;
re2c:define:YYMARKER = q;
+ re2c:define:YYCTXMARKER = r;
re2c:yyfill:enable = 0;
nul = "\000";
simple_varname = [a-zA-Z0-9_-]+;
varname = [a-zA-Z0-9_.-]+;
+ eol = "\n" | "\r\n";
+ comment = "#"[^\000\r\n]*;
- [ ]*"#"[^\000\n]*"\n" { continue; }
- [ ]*"\r\n" { token = NEWLINE; break; }
- [ ]*"\n" { token = NEWLINE; break; }
+ [ ]*comment / eol { continue; }
+ [ ]*eol { token = NEWLINE; break; }
[ ]+ { token = INDENT; break; }
"build" { token = BUILD; break; }
"pool" { token = POOL; break; }
@@ -140,12 +201,13 @@
"default" { token = DEFAULT; break; }
"=" { token = EQUALS; break; }
":" { token = COLON; break; }
+ "|@" { token = PIPEAT; break; }
"||" { token = PIPE2; break; }
"|" { token = PIPE; break; }
"include" { token = INCLUDE; break; }
"subninja" { token = SUBNINJA; break; }
varname { token = IDENT; break; }
- nul { token = TEOF; break; }
+ nul { token = (start == EndOfFile()) ? TEOF : TNUL; break; }
[^] { token = ERROR; break; }
*/
}
@@ -157,6 +219,21 @@
return token;
}
+bool Lexer::PeekIndent() {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ for (;;) {
+ start = p;
+ /*!re2c
+ [ ]*comment eol { continue; }
+ [ ]*eol { last_token_ = ofs_ = start; return false; }
+ [ ]+ { last_token_ = start; ofs_ = p; return true; }
+ [^] { last_token_ = ofs_ = start; return false; }
+ */
+ }
+}
+
bool Lexer::PeekToken(Token token) {
Token t = ReadToken();
if (t == token)
@@ -172,22 +249,21 @@
ofs_ = p;
/*!re2c
[ ]+ { continue; }
- "$\r\n" { continue; }
- "$\n" { continue; }
+ "$" eol { continue; }
nul { break; }
[^] { break; }
*/
}
}
-bool Lexer::ReadIdent(string* out) {
+bool Lexer::ReadIdent(StringPiece* out) {
const char* p = ofs_;
const char* start;
for (;;) {
start = p;
/*!re2c
varname {
- out->assign(start, p - start);
+ *out = StringPiece(start, p - start);
break;
}
[^] {
@@ -202,66 +278,30 @@
return true;
}
-bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) {
+bool Lexer::ReadBindingValue(StringPiece* out, string* err) {
const char* p = ofs_;
const char* q;
const char* start;
for (;;) {
start = p;
/*!re2c
- [^$ :\r\n|\000]+ {
- eval->AddText(StringPiece(start, p - start));
+ ( [^$\r\n\000]
+ | "$" [$: ]
+ | "$" eol
+ | "${" varname "}"
+ | "$" simple_varname )+ {
continue;
}
- "\r\n" {
- if (path)
- p = start;
+ eol {
break;
}
- [ :|\n] {
- if (path) {
- p = start;
- break;
- } else {
- if (*start == '\n')
- break;
- eval->AddText(StringPiece(start, 1));
- continue;
- }
- }
- "$$" {
- eval->AddText(StringPiece("$", 1));
- continue;
- }
- "$ " {
- eval->AddText(StringPiece(" ", 1));
- continue;
- }
- "$\r\n"[ ]* {
- continue;
- }
- "$\n"[ ]* {
- continue;
- }
- "${"varname"}" {
- eval->AddSpecial(StringPiece(start + 2, p - start - 3));
- continue;
- }
- "$"simple_varname {
- eval->AddSpecial(StringPiece(start + 1, p - start - 1));
- continue;
- }
- "$:" {
- eval->AddText(StringPiece(":", 1));
- continue;
- }
"$". {
last_token_ = start;
return Error("bad $-escape (literal $ must be written as $$)", err);
}
nul {
last_token_ = start;
- return Error("unexpected EOF", err);
+ return UnexpectedNulError(start, err);
}
[^] {
last_token_ = start;
@@ -269,10 +309,192 @@
}
*/
}
+ *out = StringPiece(ofs_, p - ofs_);
last_token_ = start;
ofs_ = p;
- if (path)
- EatWhitespace();
// Non-path strings end in newlines, so there's no whitespace to eat.
return true;
}
+
+StringPiece Lexer::PeekCanonicalPath() {
+ auto finish = [this](const char* start, const char* end) {
+ ofs_ = end;
+ EatWhitespace();
+ return StringPiece(start, end - start);
+ };
+
+ const char* p = ofs_;
+ const char* q;
+ const char* r;
+ last_token_ = ofs_;
+
+ do {
+ /*!re2c
+ canon_no_dot = [^$ :|/.\r\n\000];
+ canon_any = canon_no_dot | ".";
+ canon_piece = ( canon_no_dot | "." canon_no_dot | ".." canon_any ) canon_any*;
+ canon_pieces = canon_piece ( "/" canon_piece )*;
+
+ // The Chromium gn manifests have many paths that start with "./" but are
+ // otherwise canonical. Allow them and strip off the leading "./".
+ ( "/" | "../"* ) canon_pieces / ([ :|] | eol) { return finish(ofs_, p); }
+ "./" "../"* canon_pieces / ([ :|] | eol) { return finish(ofs_ + 2, p); }
+ [^] { break; }
+ */
+ } while (false);
+
+ return {};
+}
+
+bool Lexer::ReadPath(LexedPath* out, std::string* err) {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ for (;;) {
+ start = p;
+ /*!re2c
+ ( [^$ :|\r\n\000]
+ | "$" [$: ]
+ | "$" eol [ ]*
+ | "${" varname "}"
+ | "$" simple_varname )+ {
+ continue;
+ }
+ [ :|] | eol {
+ p = start;
+ break;
+ }
+ "$". {
+ last_token_ = start;
+ return Error("bad $-escape (literal $ must be written as $$)", err);
+ }
+ nul {
+ last_token_ = start;
+ return UnexpectedNulError(start, err);
+ }
+ [^] {
+ last_token_ = start;
+ return Error(DescribeLastError(), err);
+ }
+ */
+ }
+ *out = {};
+ out->str_ = StringPiece(ofs_, p - ofs_);
+ last_token_ = start;
+ ofs_ = p;
+ EatWhitespace();
+ return true;
+}
+
+/// Append the let binding's evaluated value to the output string. The input
+/// StringPiece must include a valid binding terminator.
+template <typename EvalVar>
+static inline void EvaluateBinding(std::string* out_append, StringPiece value,
+ EvalVar&& eval_var) {
+ auto expand = [&eval_var](const char* start, const char* end) {
+ StringPiece var(start, end - start);
+ eval_var(var);
+ };
+
+ const char* p = value.data();
+ const char* q;
+
+ for (;;) {
+ const char* start = p;
+ /*!re2c
+ [^$\r\n\000]+ { out_append->append(start, p - start); continue; }
+ "$" [$: ] { out_append->push_back(start[1]); continue; }
+ "$" eol [ ]* { continue; }
+ "${" varname "}" { expand(start + 2, p - 1); continue; }
+ "$" simple_varname { expand(start + 1, p); continue; }
+ eol { break; }
+ [^] { assert(false && "bad input in EvaluateBinding"); abort(); }
+ */
+ }
+ assert((p == value.data() + value.size()) &&
+ "bad end pos in EvaluateBinding");
+}
+
+void EvaluateBindingInScope(std::string* out_append, StringPiece value,
+ ScopePosition pos) {
+ EvaluateBinding(out_append, value,
+ [out_append, &pos](const HashedStrView& var) {
+ Scope::EvaluateVariableAtPos(out_append, var, pos);
+ });
+}
+
+bool EvaluateBindingOnRule(std::string* out_append, StringPiece value,
+ EdgeEval* edge_eval, std::string* err) {
+ bool result = true;
+ EvaluateBinding(out_append, value,
+ [out_append, edge_eval, &result, err](const HashedStrView& var) {
+ result = result && edge_eval->EvaluateVariable(out_append, var, err);
+ });
+ return result;
+}
+
+std::string EvaluateBindingForTesting(StringPiece value) {
+ std::string result;
+ EvaluateBinding(&result, value, [&result](StringPiece var) {
+ result += "[$" + var.AsString() + "]";
+ });
+ return result;
+}
+
+/// Append an evaluated path to the output string.
+///
+/// This function does not canonicalize the output. Ninja canonicalizes paths for
+/// build nodes, but not all paths (e.g. It doesn't canonicalize paths to
+/// included ninja files.)
+template <typename EvalVar>
+static inline void EvaluatePath(std::string* out_append, const LexedPath& path,
+ EvalVar&& eval_var) {
+ auto expand = [&eval_var](const char* start, const char* end) {
+ StringPiece var(start, end - start);
+ eval_var(var);
+ };
+
+ const char* p = path.str_.data();
+ const char* q;
+
+ for (;;) {
+ const char* start = p;
+ /*!re2c
+ [^$ :|\r\n\000]+ { out_append->append(start, p - start); continue; }
+ "$" [$: ] { out_append->push_back(start[1]); continue; }
+ "$" eol [ ]* { continue; }
+ "${" varname "}" { expand(start + 2, p - 1); continue; }
+ "$" simple_varname { expand(start + 1, p); continue; }
+ [ :|] | eol { p = start; break; }
+ [^] { assert(false && "bad input in EvaluatePath"); abort(); }
+ */
+ }
+ assert((p == path.str_.data() + path.str_.size()) &&
+ "bad end pos in EvaluatePath");
+}
+
+void EvaluatePathInScope(std::string* out_append, const LexedPath& path,
+ ScopePosition pos) {
+ EvaluatePath(out_append, path, [out_append, &pos](const HashedStrView& var) {
+ Scope::EvaluateVariableAtPos(out_append, var, pos);
+ });
+}
+
+void EvaluatePathOnEdge(std::string* out_append, const LexedPath& path,
+ const Edge& edge) {
+ EvaluatePath(out_append, path, [out_append, &edge](const HashedStrView& var) {
+ // First look for a binding on the edge itself, then fall back to the
+ // edge's enclosing scope.
+ if (edge.EvaluateVariableSelfOnly(out_append, var))
+ return;
+ Scope::EvaluateVariableAtPos(out_append, var, edge.pos_.scope_pos());
+ });
+}
+
+std::string EvaluatePathForTesting(const LexedPath& path) {
+ std::string result;
+ EvaluatePath(&result, path, [&result](StringPiece var) {
+ result += "[$" + var.AsString() + "]";
+ });
+ return result;
+}
diff --git a/src/lexer_test.cc b/src/lexer_test.cc
index 331d8e1..1b69bed 100644
--- a/src/lexer_test.cc
+++ b/src/lexer_test.cc
@@ -18,28 +18,30 @@
#include "test.h"
TEST(Lexer, ReadVarValue) {
- Lexer lexer("plain text $var $VaR ${x}\n");
- EvalString eval;
- string err;
- EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+ Lexer lexer("plain text $var $VaR ${x}\nmore text");
+ StringPiece uneval;
+ std::string err;
+ EXPECT_TRUE(lexer.ReadBindingValue(&uneval, &err));
EXPECT_EQ("", err);
- EXPECT_EQ("[plain text ][$var][ ][$VaR][ ][$x]",
- eval.Serialize());
+ EXPECT_EQ("plain text $var $VaR ${x}\n", uneval.AsString());
+ std::string eval = EvaluateBindingForTesting(uneval);
+ EXPECT_EQ("plain text [$var] [$VaR] [$x]", eval);
}
TEST(Lexer, ReadEvalStringEscapes) {
- Lexer lexer("$ $$ab c$: $\ncde\n");
- EvalString eval;
- string err;
- EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+ Lexer lexer("$ $$ab c$: $\ncde\nmore text");
+ StringPiece uneval;
+ std::string err;
+ EXPECT_TRUE(lexer.ReadBindingValue(&uneval, &err));
EXPECT_EQ("", err);
- EXPECT_EQ("[ $ab c: cde]",
- eval.Serialize());
+ EXPECT_EQ("$ $$ab c$: $\ncde\n", uneval.AsString());
+ std::string eval = EvaluateBindingForTesting(uneval);
+ EXPECT_EQ(" $ab c: cde", eval);
}
TEST(Lexer, ReadIdent) {
Lexer lexer("foo baR baz_123 foo-bar");
- string ident;
+ StringPiece ident;
EXPECT_TRUE(lexer.ReadIdent(&ident));
EXPECT_EQ("foo", ident);
EXPECT_TRUE(lexer.ReadIdent(&ident));
@@ -54,23 +56,71 @@
// Verify that ReadIdent includes dots in the name,
// but in an expansion $bar.dots stops at the dot.
Lexer lexer("foo.dots $bar.dots ${bar.dots}\n");
- string ident;
+ StringPiece ident;
EXPECT_TRUE(lexer.ReadIdent(&ident));
EXPECT_EQ("foo.dots", ident);
- EvalString eval;
+ StringPiece uneval;
string err;
- EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+ EXPECT_TRUE(lexer.ReadBindingValue(&uneval, &err));
EXPECT_EQ("", err);
- EXPECT_EQ("[$bar][.dots ][$bar.dots]",
- eval.Serialize());
+ EXPECT_EQ("$bar.dots ${bar.dots}\n", uneval.AsString());
+ std::string eval = EvaluateBindingForTesting(uneval);
+ EXPECT_EQ("[$bar].dots [$bar.dots]", eval);
+}
+
+TEST(Lexer, PeekCanonicalPath) {
+ auto peek_canon = [](const char* pattern) {
+ Lexer lexer(pattern);
+ return lexer.PeekCanonicalPath().AsString();
+ };
+ EXPECT_EQ("", peek_canon(""));
+ EXPECT_EQ("", peek_canon(" : "));
+ EXPECT_EQ("", peek_canon("/ : "));
+ EXPECT_EQ("", peek_canon("/foo/ : "));
+ EXPECT_EQ("", peek_canon("/foo/../bar : "));
+ EXPECT_EQ("", peek_canon("/../foo/bar : "));
+
+ EXPECT_EQ("", peek_canon("foo$$bar : "));
+ EXPECT_EQ("", peek_canon("foo${bar}baz : "));
+
+ EXPECT_EQ("/foo", peek_canon("/foo : "));
+ EXPECT_EQ("/foo/bar", peek_canon("/foo/bar : "));
+ EXPECT_EQ("foo", peek_canon("foo : "));
+ EXPECT_EQ("foo/bar", peek_canon("foo/bar : "));
+ EXPECT_EQ("../foo/bar", peek_canon("../foo/bar : "));
+ EXPECT_EQ("../../foo/bar", peek_canon("../../foo/bar : "));
+
+ EXPECT_EQ("foo", peek_canon("./foo : "));
+ EXPECT_EQ("../../foo/bar", peek_canon("./../../foo/bar : "));
+}
+
+TEST(Lexer, ReadPath) {
+ Lexer lexer("x$$y$ z$:: $bar.dots\n");
+ std::string err;
+ LexedPath uneval;
+ std::string eval;
+
+ EXPECT_TRUE(lexer.ReadPath(&uneval, &err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ("x$$y$ z$:", uneval.str_.AsString());
+ eval = EvaluatePathForTesting(uneval);
+ EXPECT_EQ("x$y z:", eval);
+
+ EXPECT_EQ(Lexer::COLON, lexer.ReadToken());
+
+ EXPECT_TRUE(lexer.ReadPath(&uneval, &err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ("$bar.dots", uneval.str_.AsString());
+ eval = EvaluatePathForTesting(uneval);
+ EXPECT_EQ("[$bar].dots", eval);
}
TEST(Lexer, Error) {
Lexer lexer("foo$\nbad $");
- EvalString eval;
+ StringPiece uneval;
string err;
- ASSERT_FALSE(lexer.ReadVarValue(&eval, &err));
+ ASSERT_FALSE(lexer.ReadBindingValue(&uneval, &err));
EXPECT_EQ("input:2: bad $-escape (literal $ must be written as $$)\n"
"bad $\n"
" ^ near here"
@@ -94,3 +144,40 @@
EXPECT_EQ(Lexer::ERROR, token);
EXPECT_EQ("tabs are not allowed, use spaces", lexer.DescribeLastError());
}
+
+TEST(Lexer, AdvanceToNextManifestChunk) {
+ auto advance = [](std::string input) -> std::string {
+ // The input string may optionally have a '^' marking where the lexer should
+ // start scanning for a chunk.
+ size_t idx = input.find_first_of('^');
+ if (idx == std::string::npos) {
+ idx = 0;
+ } else {
+ input = input.substr(0, idx) + input.substr(idx + 1);
+ }
+ idx = AdvanceToNextManifestChunk(input, idx);
+ // The return string has a '^' marking where the manifest was split.
+ return input.substr(0, idx) + "^" + input.substr(idx);
+ };
+
+ EXPECT_EQ("^", advance(""));
+ EXPECT_EQ("A\n^B\n", advance("A\nB\n"));
+ EXPECT_EQ("\n^B\n", advance("\nB\n"));
+ EXPECT_EQ("\0\0\0^"_s, advance("\0\0\0"_s));
+
+ // An LF preceded by "$" or "$\r" might be part of a line continuator, so we
+ // skip it when looking for a place to split the manifest.
+ EXPECT_EQ("A$\nB\n^C", advance("^A$\nB\nC"));
+ EXPECT_EQ("A$\nB\n^C", advance("A^$\nB\nC"));
+ EXPECT_EQ("A$\nB\n^C", advance("A$^\nB\nC"));
+ EXPECT_EQ("A$\r\nB\n^C", advance("^A$\r\nB\nC"));
+ EXPECT_EQ("A$\r\nB\n^C", advance("A^$\r\nB\nC"));
+ EXPECT_EQ("A$\r\nB\n^C", advance("A$^\r\nB\nC"));
+ EXPECT_EQ("A$\r\nB\n^C", advance("A$\r^\nB\nC"));
+
+ // Skip over blank lines and comment lines.
+ EXPECT_EQ("\n^", advance("\n"));
+ EXPECT_EQ("\n\n^A\n", advance("\n\nA\n"));
+ EXPECT_EQ("\n \n^A", advance("\n \nA"));
+ EXPECT_EQ("\n#\n^A", advance("\n#\nA"));
+}
diff --git a/src/manifest_chunk_parser.cc b/src/manifest_chunk_parser.cc
new file mode 100644
index 0000000..ff36752
--- /dev/null
+++ b/src/manifest_chunk_parser.cc
@@ -0,0 +1,433 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "manifest_chunk_parser.h"
+
+#include <assert.h>
+
+#include <thread>
+#include <unordered_map>
+
+#include "lexer.h"
+#include "metrics.h"
+#include "thread_pool.h"
+#include "util.h"
+
+namespace manifest_chunk {
+
+class ChunkParser {
+ const LoadedFile& file_;
+ Lexer lexer_;
+ const char* chunk_end_ = nullptr;
+ std::vector<ParserItem>* out_ = nullptr;
+ Clump* current_clump_ = nullptr;
+
+ Clump* MakeClump() {
+ if (current_clump_)
+ return current_clump_;
+ current_clump_ = new Clump { file_ };
+ out_->push_back(current_clump_);
+ return current_clump_;
+ }
+
+ void OutItem(ParserItem item) {
+ current_clump_ = nullptr;
+ out_->push_back(item);
+ }
+
+ bool OutError(const std::string& err) {
+ OutItem(new Error { err });
+ return false;
+ }
+
+ bool LexerError(const std::string& message);
+ bool ExpectToken(Lexer::Token expected);
+ bool ParseLet(StringPiece* key, StringPiece* value);
+ bool ParseFileInclude(bool new_scope);
+ bool ParsePool();
+ bool ParseDefault();
+ bool ParseBinding();
+ bool ParseRule();
+ bool ParseEdge();
+
+public:
+ ChunkParser(const LoadedFile& file,
+ StringPiece chunk_content,
+ std::vector<ParserItem>* out)
+ : file_(file),
+ lexer_(file.filename(), file.content(), chunk_content.data()),
+ chunk_end_(chunk_content.data() + chunk_content.size()),
+ out_(out) {}
+
+ bool ParseChunk();
+};
+
+bool ChunkParser::LexerError(const std::string& message) {
+ std::string err;
+ lexer_.Error(message, &err);
+ return OutError(err);
+}
+
+bool ChunkParser::ExpectToken(Lexer::Token expected) {
+ Lexer::Token token = lexer_.ReadToken();
+ if (token != expected) {
+ std::string message = string("expected ") + Lexer::TokenName(expected);
+ message += string(", got ") + Lexer::TokenName(token);
+ message += Lexer::TokenErrorHint(expected);
+ return LexerError(message);
+ }
+ return true;
+}
+
+bool ChunkParser::ParseLet(StringPiece* key, StringPiece* value) {
+ if (!lexer_.ReadIdent(key))
+ return LexerError("expected variable name");
+ if (!ExpectToken(Lexer::EQUALS))
+ return false;
+ std::string err;
+ if (!lexer_.ReadBindingValue(value, &err))
+ return OutError(err);
+ return true;
+}
+
+bool ChunkParser::ParseFileInclude(bool new_scope) {
+ Include* include = new Include();
+ include->new_scope_ = new_scope;
+ std::string err;
+ if (!lexer_.ReadPath(&include->path_, &err))
+ return OutError(err);
+ include->diag_pos_ = lexer_.GetLastTokenOffset();
+ if (!ExpectToken(Lexer::NEWLINE))
+ return false;
+ OutItem(include);
+ return true;
+}
+
+bool ChunkParser::ParsePool() {
+ Pool* pool = new Pool();
+
+ StringPiece name;
+ if (!lexer_.ReadIdent(&name))
+ return LexerError("expected pool name");
+ pool->name_ = name;
+
+ if (!ExpectToken(Lexer::NEWLINE))
+ return false;
+
+ pool->parse_state_.pool_name_diag_pos = lexer_.GetLastTokenOffset();
+
+ while (lexer_.PeekIndent()) {
+ StringPiece key;
+ StringPiece value;
+ if (!ParseLet(&key, &value))
+ return false;
+
+ if (key == "depth") {
+ pool->parse_state_.depth = value;
+ pool->parse_state_.depth_diag_pos = lexer_.GetLastTokenOffset();
+ } else {
+ return LexerError("unexpected variable '" + key.AsString() + "'");
+ }
+ }
+
+ if (pool->parse_state_.depth.empty())
+ return LexerError("expected 'depth =' line");
+
+ Clump* clump = MakeClump();
+ pool->pos_ = clump->AllocNextPos();
+ clump->pools_.push_back(pool);
+ return true;
+}
+
+bool ChunkParser::ParseDefault() {
+ bool first = true;
+ while (true) {
+ LexedPath path;
+ std::string err;
+ if (!lexer_.ReadPath(&path, &err))
+ return OutError(err);
+ if (path.str_.empty()) {
+ if (first)
+ return LexerError("expected target name");
+ else
+ break;
+ }
+ first = false;
+
+ DefaultTarget* target = new DefaultTarget();
+ target->parsed_path_ = std::move(path);
+ target->diag_pos_ = lexer_.GetLastTokenOffset();
+
+ Clump* clump = MakeClump();
+ target->pos_ = clump->AllocNextPos();
+ clump->default_targets_.push_back(target);
+ }
+ return ExpectToken(Lexer::NEWLINE);
+}
+
+static const HashedStrView kNinjaRequiredVersion { "ninja_required_version" };
+
+bool ChunkParser::ParseBinding() {
+ Binding* binding = new Binding();
+
+ lexer_.UnreadToken();
+ StringPiece name;
+ if (!ParseLet(&name, &binding->parsed_value_))
+ return false;
+ binding->name_ = name;
+
+ // Record a ninja_required_version binding specially. We want to do this
+ // version check ASAP in case we encounter unexpected syntax. We can't do the
+ // check immediately because the version string is evaluated and might need
+ // bindings from an earlier chunk. We could probably do this version check as
+ // part of ordinary scope setup while processing a Clump, but keeping it
+ // separate guarantees that the version check is done early enough.
+ if (binding->name_ == kNinjaRequiredVersion)
+ OutItem(new RequiredVersion { binding->parsed_value_ });
+
+ Clump* clump = MakeClump();
+ binding->pos_ = clump->AllocNextPos();
+ clump->bindings_.push_back(binding);
+ return true;
+}
+
+static const HashedStrView kRspFile { "rspfile" };
+static const HashedStrView kRspFileContent { "rspfile_content" };
+static const HashedStrView kCommand { "command" };
+
+bool ChunkParser::ParseRule() {
+ Rule* rule = new Rule();
+
+ StringPiece rule_name;
+ if (!lexer_.ReadIdent(&rule_name))
+ return LexerError("expected rule name");
+ rule->name_ = rule_name;
+
+ if (!ExpectToken(Lexer::NEWLINE))
+ return false;
+
+ rule->parse_state_.rule_name_diag_pos = lexer_.GetLastTokenOffset();
+
+ while (lexer_.PeekIndent()) {
+ StringPiece key;
+ StringPiece value;
+ if (!ParseLet(&key, &value))
+ return false;
+
+ if (!Rule::IsReservedBinding(key)) {
+ // Die on other keyvals for now; revisit if we want to add a scope here.
+ // If we allow arbitrary key values here, we'll need to revisit how cycle
+ // detection works when evaluating a rule variable.
+ return LexerError("unexpected variable '" + key.AsString() + "'");
+ }
+
+ rule->bindings_.emplace_back(std::piecewise_construct,
+ std::tuple<StringPiece>(key),
+ std::tuple<const char*, size_t>(value.data(),
+ value.size()));
+ }
+
+ if (static_cast<bool>(rule->GetBinding(kRspFile)) !=
+ static_cast<bool>(rule->GetBinding(kRspFileContent))) {
+ return LexerError("rspfile and rspfile_content need to be both specified");
+ }
+
+ if (rule->GetBinding(kCommand) == nullptr)
+ return LexerError("expected 'command =' line");
+
+ Clump* clump = MakeClump();
+ rule->pos_ = clump->AllocNextPos();
+ clump->rules_.push_back(rule);
+ return true;
+}
+
+bool ChunkParser::ParseEdge() {
+ Edge* edge = new Edge();
+
+ auto parse_path_list = [this, edge](Edge::DeferredPathList::Type type, int& count) -> bool {
+ const char* start_pos = lexer_.GetPos();
+ while (true) {
+ LexedPath path;
+ std::string err;
+ if (!lexer_.ReadPath(&path, &err))
+ return OutError(err);
+ if (path.str_.empty()) {
+ // In the initial parsing pass, the manifest's bindings aren't ready
+ // yet, so paths can't be evaluated. Rather than store the path itself
+ // (wasting memory), store just enough information to parse the path
+ // lists again once bindings are ready.
+ edge->parse_state_.deferred_path_lists.push_back({
+ start_pos, type, count,
+ });
+ return true;
+ }
+ count++;
+ }
+ };
+
+ if (!parse_path_list(Edge::DeferredPathList::OUTPUT,
+ edge->explicit_outs_))
+ return false;
+
+ // Add all implicit outs, counting how many as we go.
+ if (lexer_.PeekToken(Lexer::PIPE)) {
+ if (!parse_path_list(Edge::DeferredPathList::OUTPUT,
+ edge->implicit_outs_))
+ return false;
+ }
+
+ if (edge->explicit_outs_ + edge->implicit_outs_ == 0)
+ return LexerError("expected path");
+
+ if (!ExpectToken(Lexer::COLON))
+ return false;
+
+ StringPiece rule_name;
+ if (!lexer_.ReadIdent(&rule_name))
+ return LexerError("expected build command name");
+ edge->parse_state_.rule_name = rule_name;
+ edge->parse_state_.rule_name_diag_pos = lexer_.GetLastTokenOffset();
+
+ if (!parse_path_list(Edge::DeferredPathList::INPUT,
+ edge->explicit_deps_))
+ return false;
+
+ // Add all implicit deps, counting how many as we go.
+ if (lexer_.PeekToken(Lexer::PIPE)) {
+ if (!parse_path_list(Edge::DeferredPathList::INPUT,
+ edge->implicit_deps_))
+ return false;
+ }
+
+ // Add all order-only deps, counting how many as we go.
+ if (lexer_.PeekToken(Lexer::PIPE2)) {
+ if (!parse_path_list(Edge::DeferredPathList::INPUT,
+ edge->order_only_deps_))
+ return false;
+ }
+
+ // Add all validation deps, counting how many as we go.
+ if (lexer_.PeekToken(Lexer::PIPEAT)) {
+ if (!parse_path_list(Edge::DeferredPathList::VALIDATION,
+ edge->validation_deps_))
+ return false;
+ }
+
+
+ if (!ExpectToken(Lexer::NEWLINE))
+ return false;
+
+ while (lexer_.PeekIndent()) {
+ StringPiece key;
+ StringPiece val;
+ if (!ParseLet(&key, &val))
+ return false;
+
+ std::tuple<const char*, size_t> val_ctor_params(val.data(), val.size());
+ edge->unevaled_bindings_.emplace_back(std::piecewise_construct,
+ std::tuple<StringPiece>(key),
+ val_ctor_params);
+ }
+
+ edge->parse_state_.final_diag_pos = lexer_.GetLastTokenOffset();
+
+ Clump* clump = MakeClump();
+ edge->pos_ = clump->AllocNextPos();
+ clump->edges_.push_back(edge);
+ clump->edge_output_count_ += edge->explicit_outs_;
+ return true;
+}
+
+bool ChunkParser::ParseChunk() {
+ while (true) {
+ if (lexer_.GetPos() >= chunk_end_) {
+ assert(lexer_.GetPos() == chunk_end_ &&
+ "lexer scanned beyond the end of a manifest chunk");
+ return true;
+ }
+
+ Lexer::Token token = lexer_.ReadToken();
+ bool success = true;
+ switch (token) {
+ case Lexer::INCLUDE: success = ParseFileInclude(false); break;
+ case Lexer::SUBNINJA: success = ParseFileInclude(true); break;
+ case Lexer::POOL: success = ParsePool(); break;
+ case Lexer::DEFAULT: success = ParseDefault(); break;
+ case Lexer::IDENT: success = ParseBinding(); break;
+ case Lexer::RULE: success = ParseRule(); break;
+ case Lexer::BUILD: success = ParseEdge(); break;
+ case Lexer::NEWLINE: break;
+ case Lexer::ERROR: return LexerError(lexer_.DescribeLastError());
+ case Lexer::TNUL: return LexerError("unexpected NUL byte");
+ case Lexer::TEOF:
+ assert(false && "EOF should have been detected before reading a token");
+ break;
+ default:
+ return LexerError(std::string("unexpected ") + Lexer::TokenName(token));
+ }
+ if (!success) return false;
+ }
+ return false; // not reached
+}
+
+void ParseChunk(const LoadedFile& file, StringPiece chunk_content,
+ std::vector<ParserItem>* out) {
+ ChunkParser parser(file, chunk_content, out);
+ if (!parser.ParseChunk()) {
+ assert(!out->empty());
+ assert(out->back().kind == ParserItem::kError);
+ assert(!out->back().u.error->msg_.empty());
+ }
+}
+
+std::vector<StringPiece> SplitManifestIntoChunks(StringPiece input) {
+ METRIC_RECORD(".ninja load : split manifest");
+
+ // The lexer requires that a NUL character appear after the file's content.
+ // Make the NUL implicit for the rest of the manifest parsing.
+ //
+ // In general, parsing a manifest chunk examines the terminating text after
+ // the chunk's explicit end. For example, suppose we have two consecutive
+ // chunks:
+ // - "build foo: gen\n pool = mypool\n"
+ // - "build bar: gen\n"
+ // The parser for the "foo" chunk will examine the start of "build" from the
+ // "bar" chunk when it looks for another indented edge binding.
+ assert(!input.empty() && input.back() == '\0' &&
+ "input lacks a NUL terminator");
+ input.remove_suffix(1);
+
+ // For efficiency, avoid splitting small files. kChunkMinSize can be lowered
+ // to 0 for testing.
+ const size_t kChunkMinSize = 1024 * 1024;
+
+ const size_t chunk_count = GetOptimalThreadPoolJobCount();
+ const size_t chunk_size = std::max(kChunkMinSize,
+ input.size() / chunk_count + 1);
+
+ std::vector<StringPiece> result;
+ result.reserve(chunk_count);
+
+ size_t start = 0;
+ while (start < input.size()) {
+ size_t next = std::min(start + chunk_size, input.size());
+ next = AdvanceToNextManifestChunk(input, next);
+ result.push_back(input.substr(start, next - start));
+ start = next;
+ }
+
+ return result;
+}
+
+} // namespace manifest_chunk
diff --git a/src/manifest_chunk_parser.h b/src/manifest_chunk_parser.h
new file mode 100644
index 0000000..665a805
--- /dev/null
+++ b/src/manifest_chunk_parser.h
@@ -0,0 +1,112 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_MANIFEST_CHUNK_PARSER_H_
+#define NINJA_MANIFEST_CHUNK_PARSER_H_
+
+#include <stdlib.h>
+
+#include <string>
+#include <vector>
+
+#include "disk_interface.h"
+#include "eval_env.h"
+#include "graph.h"
+#include "state.h"
+#include "string_piece.h"
+
+struct Lexer;
+
+namespace manifest_chunk {
+
+struct Error {
+ std::string msg_;
+};
+
+struct RequiredVersion {
+ StringPiece version_;
+};
+
+struct Include {
+ LexedPath path_;
+ bool new_scope_ = false;
+ size_t diag_pos_ = 0;
+};
+
+struct DefaultTarget {
+ RelativePosition pos_;
+ LexedPath parsed_path_;
+ size_t diag_pos_ = 0;
+};
+
+/// A group of manifest declarations generated while parsing a chunk.
+struct Clump {
+ Clump(const LoadedFile& file) : file_(file) {}
+
+ const LoadedFile& file_;
+ BasePosition pos_;
+
+ std::vector<Binding*> bindings_;
+ std::vector<Rule*> rules_;
+ std::vector<Pool*> pools_;
+ std::vector<Edge*> edges_;
+ std::vector<DefaultTarget*> default_targets_;
+
+ /// A count of non-implicit outputs across all edges.
+ size_t edge_output_count_ = 0;
+
+ DeclIndex decl_count() const { return next_index_; }
+
+ /// Allocate an index within the clump. Once the parallelized chunk parsing is
+ /// finished, each clump's base position will be computed, giving every clump
+ /// item both a DFS "depth-first-search" position and a position within the
+ /// tree of scopes.
+ RelativePosition AllocNextPos() { return { &pos_, next_index_++ }; }
+
+private:
+ DeclIndex next_index_ = 0;
+};
+
+/// This class could be replaced with std::variant from C++17.
+struct ParserItem {
+ enum Kind {
+ kError, kRequiredVersion, kInclude, kClump
+ };
+
+ Kind kind;
+
+ union {
+ Error* error;
+ RequiredVersion* required_version;
+ Include* include;
+ Clump* clump;
+ } u;
+
+ ParserItem(Error* val) : kind(kError) { u.error = val; }
+ ParserItem(RequiredVersion* val) : kind(kRequiredVersion) { u.required_version = val; }
+ ParserItem(Include* val) : kind(kInclude) { u.include = val; }
+ ParserItem(Clump* val) : kind(kClump) { u.clump = val; }
+};
+
+/// Parse a chunk of a manifest. If the parser encounters a syntactic error, the
+/// final item will be an error object.
+void ParseChunk(const LoadedFile& file, StringPiece chunk_content,
+ std::vector<ParserItem>* out);
+
+/// Split the input into chunks of declarations.
+std::vector<StringPiece> SplitManifestIntoChunks(StringPiece input);
+
+} // namespace manifest_chunk
+
+#endif // NINJA_MANIFEST_CHUNK_PARSER_H_
diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc
index 2011368..ad61e20 100644
--- a/src/manifest_parser.cc
+++ b/src/manifest_parser.cc
@@ -1,4 +1,4 @@
-// Copyright 2011 Google Inc. All Rights Reserved.
+// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -14,417 +14,618 @@
#include "manifest_parser.h"
-#include <stdio.h>
-#include <stdlib.h>
+#include <assert.h>
+
+#include <unordered_map>
#include <vector>
-#include "graph.h"
+#include "disk_interface.h"
+#include "eval_env.h"
+#include "lexer.h"
+#include "metrics.h"
+#include "parallel_map.h"
#include "state.h"
#include "util.h"
#include "version.h"
+#include "manifest_chunk_parser.h"
-ManifestParser::ManifestParser(State* state, FileReader* file_reader,
- ManifestParserOptions options)
- : Parser(state, file_reader),
- options_(options), quiet_(false) {
- env_ = &state->bindings_;
+using manifest_chunk::Clump;
+using manifest_chunk::DefaultTarget;
+using manifest_chunk::Include;
+using manifest_chunk::ParserItem;
+using manifest_chunk::RequiredVersion;
+
+namespace {
+
+static bool DecorateError(const LoadedFile& file, size_t file_offset,
+ const std::string& message, std::string* err) {
+ assert(file_offset <= file.content().size());
+ return DecorateErrorWithLocation(file.filename(), file.content().data(),
+ file_offset, message, err);
}
-bool ManifestParser::Parse(const string& filename, const string& input,
- string* err) {
- lexer_.Start(filename, input);
+/// Split a single manifest file into chunks, parse the chunks in parallel, and
+/// return the resulting parser output.
+static std::vector<ParserItem> ParseManifestChunks(const LoadedFile& file,
+ ThreadPool* thread_pool) {
+ std::vector<ParserItem> result;
+ const std::vector<StringPiece> chunk_views =
+ manifest_chunk::SplitManifestIntoChunks(file.content_with_nul());
- for (;;) {
- Lexer::Token token = lexer_.ReadToken();
- switch (token) {
- case Lexer::POOL:
- if (!ParsePool(err))
- return false;
- break;
- case Lexer::BUILD:
- if (!ParseEdge(err))
- return false;
- break;
- case Lexer::RULE:
- if (!ParseRule(err))
- return false;
- break;
- case Lexer::DEFAULT:
- if (!ParseDefault(err))
- return false;
- break;
- case Lexer::IDENT: {
- lexer_.UnreadToken();
- string name;
- EvalString let_value;
- if (!ParseLet(&name, &let_value, err))
- return false;
- string value = let_value.Evaluate(env_);
- // Check ninja_required_version immediately so we can exit
- // before encountering any syntactic surprises.
- if (name == "ninja_required_version")
- CheckNinjaVersion(value);
- env_->AddBinding(name, value);
- break;
- }
- case Lexer::INCLUDE:
- if (!ParseFileInclude(false, err))
- return false;
- break;
- case Lexer::SUBNINJA:
- if (!ParseFileInclude(true, err))
- return false;
- break;
- case Lexer::ERROR: {
- return lexer_.Error(lexer_.DescribeLastError(), err);
- }
- case Lexer::TEOF:
- return true;
- case Lexer::NEWLINE:
- break;
- default:
- return lexer_.Error(string("unexpected ") + Lexer::TokenName(token),
- err);
+ METRIC_RECORD(".ninja load : parse chunks");
+
+ for (std::vector<ParserItem>& chunk_items :
+ ParallelMap(thread_pool, chunk_views, [&file](StringPiece view) {
+ std::vector<ParserItem> chunk_items;
+ manifest_chunk::ParseChunk(file, view, &chunk_items);
+ return chunk_items;
+ })) {
+ std::move(chunk_items.begin(), chunk_items.end(),
+ std::back_inserter(result));
+ }
+ return result;
+}
+
+static void ReserveSpaceInScopeTables(Scope* scope,
+ const std::vector<ParserItem>& items) {
+ // Reserve extra space in the scope's bindings/rules hash tables.
+ METRIC_RECORD(".ninja load : alloc scope tables");
+ size_t new_bindings = 0;
+ size_t new_rules = 0;
+ for (const auto& item : items) {
+ if (item.kind == ParserItem::kClump) {
+ Clump* clump = item.u.clump;
+ new_bindings += clump->bindings_.size();
+ new_rules += clump->rules_.size();
}
}
- return false; // not reached
+ scope->ReserveTableSpace(new_bindings, new_rules);
}
+struct ManifestFileSet {
+ ManifestFileSet(FileReader* file_reader) : file_reader_(file_reader) {}
-bool ManifestParser::ParsePool(string* err) {
- string name;
- if (!lexer_.ReadIdent(&name))
- return lexer_.Error("expected pool name", err);
+ bool LoadFile(const std::string& filename, const LoadedFile** result,
+ std::string* err);
- if (!ExpectToken(Lexer::NEWLINE, err))
+ ~ManifestFileSet() {
+ // It can take a surprising long time to unmap all the manifest files
+ // (e.g. ~70 ms for Android, 25 ms for Chromium).
+ METRIC_RECORD(".ninja load : unload files");
+ loaded_files_.clear();
+ }
+
+private:
+ FileReader *file_reader_ = nullptr;
+
+ /// Keep all the manifests in memory until all the manifest parsing work is
+ /// complete, because intermediate parsing results (i.e. StringPiece) may
+ /// point into the loaded files.
+ std::vector<std::unique_ptr<LoadedFile>> loaded_files_;
+};
+
+bool ManifestFileSet::LoadFile(const std::string& filename,
+ const LoadedFile** result,
+ std::string* err) {
+ std::unique_ptr<LoadedFile> loaded_file;
+ if (file_reader_->LoadFile(filename, &loaded_file, err) != FileReader::Okay) {
+ *err = "loading '" + filename + "': " + *err;
return false;
-
- if (state_->LookupPool(name) != NULL)
- return lexer_.Error("duplicate pool '" + name + "'", err);
-
- int depth = -1;
-
- while (lexer_.PeekToken(Lexer::INDENT)) {
- string key;
- EvalString value;
- if (!ParseLet(&key, &value, err))
- return false;
-
- if (key == "depth") {
- string depth_string = value.Evaluate(env_);
- depth = atol(depth_string.c_str());
- if (depth < 0)
- return lexer_.Error("invalid pool depth", err);
- } else {
- return lexer_.Error("unexpected variable '" + key + "'", err);
- }
}
-
- if (depth < 0)
- return lexer_.Error("expected 'depth =' line", err);
-
- state_->AddPool(new Pool(name, depth));
+ loaded_files_.push_back(std::move(loaded_file));
+ *result = loaded_files_.back().get();
return true;
}
+struct DfsParser {
+ DfsParser(ManifestFileSet* file_set, State* state, ThreadPool* thread_pool)
+ : file_set_(file_set), state_(state), thread_pool_(thread_pool) {}
-bool ManifestParser::ParseRule(string* err) {
- string name;
- if (!lexer_.ReadIdent(&name))
- return lexer_.Error("expected rule name", err);
+private:
+ void HandleRequiredVersion(const RequiredVersion& item, Scope* scope);
+ bool HandleInclude(const Include& item, const LoadedFile& file, Scope* scope,
+ const LoadedFile** child_file, Scope** child_scope,
+ std::string* err);
+ bool HandlePool(Pool* pool, const LoadedFile& file, std::string* err);
+ bool HandleClump(Clump* clump, const LoadedFile& file, Scope* scope,
+ std::string* err);
- if (!ExpectToken(Lexer::NEWLINE, err))
- return false;
+public:
+ /// Load the tree of manifest files and do initial parsing of all the chunks.
+ /// This function always runs on the main thread.
+ bool LoadManifestTree(const LoadedFile& file, Scope* scope,
+ std::vector<Clump*>* out_clumps, std::string* err);
- if (env_->LookupRuleCurrentScope(name) != NULL)
- return lexer_.Error("duplicate rule '" + name + "'", err);
+private:
+ ManifestFileSet* file_set_;
+ State* state_;
+ ThreadPool* thread_pool_;
+};
- Rule* rule = new Rule(name); // XXX scoped_ptr
+void DfsParser::HandleRequiredVersion(const RequiredVersion& item,
+ Scope* scope) {
+ std::string version;
+ EvaluateBindingInScope(&version, item.version_,
+ scope->GetCurrentEndOfScope());
+ CheckNinjaVersion(version);
+}
- while (lexer_.PeekToken(Lexer::INDENT)) {
- string key;
- EvalString value;
- if (!ParseLet(&key, &value, err))
- return false;
-
- if (Rule::IsReservedBinding(key)) {
- rule->AddBinding(key, value);
- } else {
- // Die on other keyvals for now; revisit if we want to add a
- // scope here.
- return lexer_.Error("unexpected variable '" + key + "'", err);
- }
+bool DfsParser::HandleInclude(const Include& include, const LoadedFile& file,
+ Scope* scope, const LoadedFile** child_file,
+ Scope** child_scope, std::string* err) {
+ std::string path;
+ EvaluatePathInScope(&path, include.path_, scope->GetCurrentEndOfScope());
+ if (!file_set_->LoadFile(path, child_file, err)) {
+ return DecorateError(file, include.diag_pos_, std::string(*err), err);
}
-
- if (rule->bindings_["rspfile"].empty() !=
- rule->bindings_["rspfile_content"].empty()) {
- return lexer_.Error("rspfile and rspfile_content need to be "
- "both specified", err);
+ if (include.new_scope_) {
+ *child_scope = new Scope(scope->GetCurrentEndOfScope());
+ } else {
+ *child_scope = scope;
}
-
- if (rule->bindings_["command"].empty())
- return lexer_.Error("expected 'command =' line", err);
-
- env_->AddRule(rule);
return true;
}
-bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) {
- if (!lexer_.ReadIdent(key))
- return lexer_.Error("expected variable name", err);
- if (!ExpectToken(Lexer::EQUALS, err))
- return false;
- if (!lexer_.ReadVarValue(value, err))
- return false;
+bool DfsParser::HandlePool(Pool* pool, const LoadedFile& file,
+ std::string* err) {
+ std::string depth_string;
+ EvaluateBindingInScope(&depth_string, pool->parse_state_.depth,
+ pool->pos_.scope_pos());
+ pool->depth_ = atol(depth_string.c_str());
+ if (pool->depth_ < 0) {
+ return DecorateError(file, pool->parse_state_.depth_diag_pos,
+ "invalid pool depth", err);
+ }
+ if (!state_->AddPool(pool)) {
+ return DecorateError(file, pool->parse_state_.pool_name_diag_pos,
+ "duplicate pool '" + pool->name() + "'", err);
+ }
return true;
}
-bool ManifestParser::ParseDefault(string* err) {
- EvalString eval;
- if (!lexer_.ReadPath(&eval, err))
- return false;
- if (eval.empty())
- return lexer_.Error("expected target name", err);
-
- do {
- string path = eval.Evaluate(env_);
- string path_err;
- uint64_t slash_bits; // Unused because this only does lookup.
- if (!CanonicalizePath(&path, &slash_bits, &path_err))
- return lexer_.Error(path_err, err);
- if (!state_->AddDefault(path, &path_err))
- return lexer_.Error(path_err, err);
-
- eval.Clear();
- if (!lexer_.ReadPath(&eval, err))
- return false;
- } while (!eval.empty());
-
- if (!ExpectToken(Lexer::NEWLINE, err))
- return false;
-
- return true;
-}
-
-bool ManifestParser::ParseEdge(string* err) {
- vector<EvalString> ins, outs;
-
+bool DfsParser::HandleClump(Clump* clump, const LoadedFile& file, Scope* scope,
+ std::string* err) {
+ METRIC_RECORD(".ninja load : scope setup");
+ // Allocate DFS and scope positions for the clump.
+ clump->pos_.scope = scope->AllocDecls(clump->decl_count());
+ clump->pos_.dfs_location = state_->AllocDfsLocation(clump->decl_count());
{
- EvalString out;
- if (!lexer_.ReadPath(&out, err))
- return false;
- while (!out.empty()) {
- outs.push_back(out);
-
- out.Clear();
- if (!lexer_.ReadPath(&out, err))
- return false;
+ METRIC_RECORD(".ninja load : scope setup : bindings");
+ for (Binding* binding : clump->bindings_) {
+ scope->AddBinding(binding);
}
}
-
- // Add all implicit outs, counting how many as we go.
- int implicit_outs = 0;
- if (lexer_.PeekToken(Lexer::PIPE)) {
- for (;;) {
- EvalString out;
- if (!lexer_.ReadPath(&out, err))
- return err;
- if (out.empty())
- break;
- outs.push_back(out);
- ++implicit_outs;
- }
- }
-
- if (outs.empty())
- return lexer_.Error("expected path", err);
-
- if (!ExpectToken(Lexer::COLON, err))
- return false;
-
- string rule_name;
- if (!lexer_.ReadIdent(&rule_name))
- return lexer_.Error("expected build command name", err);
-
- const Rule* rule = env_->LookupRule(rule_name);
- if (!rule)
- return lexer_.Error("unknown build rule '" + rule_name + "'", err);
-
- for (;;) {
- // XXX should we require one path here?
- EvalString in;
- if (!lexer_.ReadPath(&in, err))
- return false;
- if (in.empty())
- break;
- ins.push_back(in);
- }
-
- // Add all implicit deps, counting how many as we go.
- int implicit = 0;
- if (lexer_.PeekToken(Lexer::PIPE)) {
- for (;;) {
- EvalString in;
- if (!lexer_.ReadPath(&in, err))
- return err;
- if (in.empty())
- break;
- ins.push_back(in);
- ++implicit;
- }
- }
-
- // Add all order-only deps, counting how many as we go.
- int order_only = 0;
- if (lexer_.PeekToken(Lexer::PIPE2)) {
- for (;;) {
- EvalString in;
- if (!lexer_.ReadPath(&in, err))
- return false;
- if (in.empty())
- break;
- ins.push_back(in);
- ++order_only;
- }
- }
-
- if (!ExpectToken(Lexer::NEWLINE, err))
- return false;
-
- // Bindings on edges are rare, so allocate per-edge envs only when needed.
- bool has_indent_token = lexer_.PeekToken(Lexer::INDENT);
- BindingEnv* env = has_indent_token ? new BindingEnv(env_) : env_;
- while (has_indent_token) {
- string key;
- EvalString val;
- if (!ParseLet(&key, &val, err))
- return false;
-
- env->AddBinding(key, val.Evaluate(env_));
- has_indent_token = lexer_.PeekToken(Lexer::INDENT);
- }
-
- Edge* edge = state_->AddEdge(rule);
- edge->env_ = env;
-
- string pool_name = edge->GetBinding("pool");
- if (!pool_name.empty()) {
- Pool* pool = state_->LookupPool(pool_name);
- if (pool == NULL)
- return lexer_.Error("unknown pool name '" + pool_name + "'", err);
- edge->pool_ = pool;
- }
-
- edge->outputs_.reserve(outs.size());
- for (size_t i = 0, e = outs.size(); i != e; ++i) {
- string path = outs[i].Evaluate(env);
- string path_err;
- uint64_t slash_bits;
- if (!CanonicalizePath(&path, &slash_bits, &path_err))
- return lexer_.Error(path_err, err);
- if (!state_->AddOut(edge, path, slash_bits)) {
- if (options_.dupe_edge_action_ == kDupeEdgeActionError) {
- lexer_.Error("multiple rules generate " + path + " [-w dupbuild=err]",
- err);
- return false;
- } else {
- if (!quiet_) {
- Warning("multiple rules generate %s. "
- "builds involving this target will not be correct; "
- "continuing anyway [-w dupbuild=warn]",
- path.c_str());
- }
- if (e - i <= static_cast<size_t>(implicit_outs))
- --implicit_outs;
+ {
+ METRIC_RECORD(".ninja load : scope setup : rules");
+ for (Rule* rule : clump->rules_) {
+ if (!scope->AddRule(rule)) {
+ return DecorateError(file, rule->parse_state_.rule_name_diag_pos,
+ "duplicate rule '" + rule->name() + "'", err);
}
}
}
- if (edge->outputs_.empty()) {
- // All outputs of the edge are already created by other edges. Don't add
- // this edge. Do this check before input nodes are connected to the edge.
- state_->edges_.pop_back();
- delete edge;
- return true;
+ for (Pool* pool : clump->pools_) {
+ if (!HandlePool(pool, file, err)) {
+ return false;
+ }
}
- edge->implicit_outs_ = implicit_outs;
+ return true;
+}
- edge->inputs_.reserve(ins.size());
- for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) {
- string path = i->Evaluate(env);
- string path_err;
- uint64_t slash_bits;
- if (!CanonicalizePath(&path, &slash_bits, &path_err))
- return lexer_.Error(path_err, err);
- state_->AddIn(edge, path, slash_bits);
+bool DfsParser::LoadManifestTree(const LoadedFile& file, Scope* scope,
+ std::vector<Clump*>* out_clumps,
+ std::string* err) {
+ std::vector<ParserItem> items = ParseManifestChunks(file, thread_pool_);
+ ReserveSpaceInScopeTables(scope, items);
+
+ // With the chunks parsed, do a depth-first parse of the ninja manifest using
+ // the results of the parallel parse.
+ for (const auto& item : items) {
+ switch (item.kind) {
+ case ParserItem::kError:
+ *err = item.u.error->msg_;
+ return false;
+
+ case ParserItem::kRequiredVersion:
+ HandleRequiredVersion(*item.u.required_version, scope);
+ break;
+
+ case ParserItem::kInclude: {
+ const Include& include = *item.u.include;
+ const LoadedFile* child_file = nullptr;
+ Scope* child_scope = nullptr;
+ if (!HandleInclude(include, file, scope, &child_file, &child_scope, err))
+ return false;
+ if (!LoadManifestTree(*child_file, child_scope, out_clumps, err))
+ return false;
+ break;
+ }
+
+ case ParserItem::kClump:
+ if (!HandleClump(item.u.clump, file, scope, err))
+ return false;
+ out_clumps->push_back(item.u.clump);
+ break;
+
+ default:
+ assert(false && "unrecognized kind of ParserItem");
+ abort();
+ }
}
- edge->implicit_deps_ = implicit;
- edge->order_only_deps_ = order_only;
+ return true;
+}
+
+/// Parse an edge's path and add it to a vector on the Edge object.
+static inline bool AddPathToEdge(State* state, const Edge& edge,
+ std::vector<Node*>* out_vec,
+ const LoadedFile& file, Lexer& lexer,
+ std::string* err) {
+ HashedStrView key;
+ uint64_t slash_bits = 0;
+
+ StringPiece canon_path = lexer.PeekCanonicalPath();
+ if (!canon_path.empty()) {
+ key = canon_path;
+ } else {
+ LexedPath path;
+ if (!lexer.ReadPath(&path, err) || path.str_.empty()) {
+ assert(false && "manifest file apparently changed during parsing");
+ abort();
+ }
+
+ thread_local std::string tls_work_buf;
+ std::string& work_buf = tls_work_buf;
+ work_buf.clear();
+ EvaluatePathOnEdge(&work_buf, path, edge);
+
+ std::string path_err;
+ if (!CanonicalizePath(&work_buf, &slash_bits, &path_err)) {
+ return DecorateError(file, edge.parse_state_.final_diag_pos, path_err,
+ err);
+ }
+ key = work_buf;
+ }
+
+ Node* node = state->GetNode(key, 0);
+ node->UpdateFirstReference(edge.dfs_location(), slash_bits);
+ out_vec->push_back(node);
+ return true;
+}
+
+static const HashedStrView kPool { "pool" };
+static const HashedStrView kDeps { "deps" };
+static const HashedStrView kDyndep { "dyndep" };
+
+struct ManifestLoader {
+private:
+ State* const state_ = nullptr;
+ ThreadPool* const thread_pool_ = nullptr;
+ const ManifestParserOptions options_;
+ const bool quiet_ = false;
+
+ bool AddEdgeToGraph(Edge* edge, const LoadedFile& file, std::string* err);
+
+ /// This function runs on a worker thread and adds a clump's declarations to
+ /// the build graph.
+ bool FinishAddingClumpToGraph(Clump* clump, std::string* err);
+
+ bool FinishLoading(const std::vector<Clump*>& clumps, std::string* err);
+
+public:
+ ManifestLoader(State* state, ThreadPool* thread_pool,
+ ManifestParserOptions options, bool quiet)
+ : state_(state), thread_pool_(thread_pool), options_(options),
+ quiet_(quiet) {}
+
+ bool Load(ManifestFileSet* file_set, const LoadedFile& root_manifest,
+ std::string* err);
+};
+
+bool ManifestLoader::AddEdgeToGraph(Edge* edge, const LoadedFile& file,
+ std::string* err) {
+
+ const ScopePosition edge_pos = edge->pos_.scope_pos();
+
+ // Look up the edge's rule.
+ edge->rule_ = Scope::LookupRuleAtPos(edge->parse_state_.rule_name, edge_pos);
+ if (edge->rule_ == nullptr) {
+ std::string msg = "unknown build rule '" +
+ edge->parse_state_.rule_name.AsString() + "'";
+ return DecorateError(file, edge->parse_state_.rule_name_diag_pos, msg, err);
+ }
+
+ // Now that the edge's bindings are available, check whether the edge has a
+ // pool. This check requires the full edge+rule evaluation system.
+ std::string pool_name;
+ if (!edge->EvaluateVariable(&pool_name, kPool, err, EdgeEval::kParseTime))
+ return false;
+ if (pool_name.empty()) {
+ edge->pool_ = &State::kDefaultPool;
+ } else {
+ edge->pool_ = state_->LookupPoolAtPos(pool_name, edge->pos_.dfs_location());
+ if (edge->pool_ == nullptr) {
+ return DecorateError(file, edge->parse_state_.final_diag_pos,
+ "unknown pool name '" + pool_name + "'", err);
+ }
+ }
+
+ edge->outputs_.reserve(edge->explicit_outs_ + edge->implicit_outs_);
+ edge->inputs_.reserve(edge->explicit_deps_ + edge->implicit_deps_ +
+ edge->order_only_deps_);
+ edge->validations_.reserve(edge->validation_deps_);
+
+ // Add the input and output nodes. We already lexed them in the first pass,
+ // but we couldn't add them because scope bindings weren't available. To save
+ // memory, the first pass only recorded the lexer position of each category
+ // of input/output nodes, rather than each path's location.
+ Lexer lexer(file.filename(), file.content(), file.content().data());
+ for (const Edge::DeferredPathList& path_list :
+ edge->parse_state_.deferred_path_lists) {
+ std::vector<Node*>* vec =
+ path_list.type == Edge::DeferredPathList::INPUT ? &edge->inputs_ :
+ path_list.type == Edge::DeferredPathList::OUTPUT ? &edge->outputs_ :
+ &edge->validations_;
+ lexer.ResetPos(path_list.lexer_pos);
+ for (int i = 0; i < path_list.count; ++i) {
+ if (!AddPathToEdge(state_, *edge, vec, file, lexer, err))
+ return false;
+ }
+ // Verify that there are no more paths to parse.
+ LexedPath path;
+ if (!lexer.ReadPath(&path, err) || !path.str_.empty()) {
+ assert(false && "manifest file apparently changed during parsing");
+ abort();
+ }
+ }
+
+ // This compatibility mode filters nodes from the edge->inputs_ list; do it
+ // before linking the edge inputs and nodes.
if (options_.phony_cycle_action_ == kPhonyCycleActionWarn &&
edge->maybe_phonycycle_diagnostic()) {
- // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements
- // that reference themselves. Ninja used to tolerate these in the
- // build graph but that has since been fixed. Filter them out to
- // support users of those old CMake versions.
+ // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements that
+ // reference themselves. Ninja used to tolerate these in the build graph
+ // but that has since been fixed. Filter them out to support users of those
+ // old CMake versions.
Node* out = edge->outputs_[0];
- vector<Node*>::iterator new_end =
- remove(edge->inputs_.begin(), edge->inputs_.end(), out);
+ std::vector<Node*>::iterator new_end =
+ std::remove(edge->inputs_.begin(), edge->inputs_.end(), out);
if (new_end != edge->inputs_.end()) {
edge->inputs_.erase(new_end, edge->inputs_.end());
+ --edge->explicit_deps_;
if (!quiet_) {
- Warning("phony target '%s' names itself as an input; "
- "ignoring [-w phonycycle=warn]",
- out->path().c_str());
+ Warning("phony target '%s' names itself as an input; ignoring "
+ "[-w phonycycle=warn]", out->path().c_str());
}
}
}
// Multiple outputs aren't (yet?) supported with depslog.
- string deps_type = edge->GetBinding("deps");
- if (!deps_type.empty() && edge->outputs_.size() > 1) {
- return lexer_.Error("multiple outputs aren't (yet?) supported by depslog; "
- "bring this up on the mailing list if it affects you",
- err);
+ std::string deps_type;
+ if (!edge->EvaluateVariable(&deps_type, kDeps, err, EdgeEval::kParseTime))
+ return false;
+ if (!deps_type.empty() && edge->outputs_.size() - edge->implicit_outs_ > 1) {
+ return DecorateError(file, edge->parse_state_.final_diag_pos,
+ "multiple outputs aren't (yet?) supported by depslog; "
+ "bring this up on the mailing list if it affects you",
+ err);
}
// Lookup, validate, and save any dyndep binding. It will be used later
// to load generated dependency information dynamically, but it must
// be one of our manifest-specified inputs.
- string dyndep = edge->GetUnescapedDyndep();
+ std::string dyndep;
+ if (!edge->EvaluateVariable(&dyndep, kDyndep, err, EdgeEval::kParseTime))
+ return false;
if (!dyndep.empty()) {
uint64_t slash_bits;
if (!CanonicalizePath(&dyndep, &slash_bits, err))
return false;
- edge->dyndep_ = state_->GetNode(dyndep, slash_bits);
+ edge->dyndep_ = state_->GetNode(dyndep, 0);
edge->dyndep_->set_dyndep_pending(true);
vector<Node*>::iterator dgi =
std::find(edge->inputs_.begin(), edge->inputs_.end(), edge->dyndep_);
if (dgi == edge->inputs_.end()) {
- return lexer_.Error("dyndep '" + dyndep + "' is not an input", err);
+ return DecorateError(file, edge->parse_state_.final_diag_pos,
+ "dyndep '" + dyndep + "' is not an input", err);
}
}
return true;
}
-bool ManifestParser::ParseFileInclude(bool new_scope, string* err) {
- EvalString eval;
- if (!lexer_.ReadPath(&eval, err))
- return false;
- string path = eval.Evaluate(env_);
+bool ManifestLoader::FinishAddingClumpToGraph(Clump* clump, std::string* err) {
+ std::string work_buf;
- ManifestParser subparser(state_, file_reader_, options_);
- if (new_scope) {
- subparser.env_ = new BindingEnv(env_);
- } else {
- subparser.env_ = env_;
+ // Precompute all binding values. Discard each evaluated string -- we just
+ // need to make sure each binding's value isn't coming from the mmap'ed
+ // manifest anymore.
+ for (Binding* binding : clump->bindings_) {
+ work_buf.clear();
+ binding->Evaluate(&work_buf);
}
- if (!subparser.Load(path, err, &lexer_))
- return false;
-
- if (!ExpectToken(Lexer::NEWLINE, err))
- return false;
+ for (Edge* edge : clump->edges_) {
+ if (!AddEdgeToGraph(edge, clump->file_, err))
+ return false;
+ }
return true;
}
+
+bool ManifestLoader::FinishLoading(const std::vector<Clump*>& clumps,
+ std::string* err) {
+ {
+ // Most of this pass's time is spent adding the edges. (i.e. The time spent
+ // evaluating the bindings is negligible.)
+ METRIC_RECORD(".ninja load : edge setup");
+
+ size_t output_count = 0;
+ for (Clump* clump : clumps)
+ output_count += clump->edge_output_count_;
+
+ // Construct the initial graph of input/output nodes. Select an initial size
+ // that's likely to keep the number of collisions low. The number of edges'
+ // non-implicit outputs is a decent enough proxy for the final number of
+ // nodes. (I see acceptable performance even with a much lower number of
+ // buckets, e.g. 100 times fewer.)
+ state_->paths_.reserve(state_->paths_.size() + output_count * 3);
+
+ if (!PropagateError(err, ParallelMap(thread_pool_, clumps,
+ [this](Clump* clump) {
+ std::string err;
+ FinishAddingClumpToGraph(clump, &err);
+ return err;
+ }))) {
+ return false;
+ }
+ }
+ {
+ // Record the in-edge for each node that's built by an edge. Detect
+ // duplicate edges.
+ //
+ // With dupbuild=warn (the default until 1.9.0), when two edges generate the
+ // same node, remove the duplicate node from the output list of the later
+ // edge. If all of an edge's outputs are removed, remove the edge from the
+ // graph.
+ METRIC_RECORD(".ninja load : link edge outputs");
+ for (Clump* clump : clumps) {
+ for (size_t edge_idx = 0; edge_idx < clump->edges_.size(); ) {
+ // Scan all Edge outputs and link them to the Node objects.
+ Edge* edge = clump->edges_[edge_idx];
+ for (size_t i = 0; i < edge->outputs_.size(); ) {
+ Node* output = edge->outputs_[i];
+ if (output->in_edge() == nullptr) {
+ output->set_in_edge(edge);
+ ++i;
+ continue;
+ }
+ // Two edges produce the same output node.
+ if (options_.dupe_edge_action_ == kDupeEdgeActionError) {
+ return DecorateError(clump->file_,
+ edge->parse_state_.final_diag_pos,
+ "multiple rules generate " + output->path() +
+ " [-w dupbuild=err]", err);
+ } else {
+ if (!quiet_) {
+ Warning("multiple rules generate %s. "
+ "builds involving this target will not be correct; "
+ "continuing anyway [-w dupbuild=warn]",
+ output->path().c_str());
+ }
+ if (edge->is_implicit_out(i))
+ --edge->implicit_outs_;
+ else
+ --edge->explicit_outs_;
+ edge->outputs_.erase(edge->outputs_.begin() + i);
+ }
+ }
+ if (edge->outputs_.empty()) {
+ // All outputs of the edge are already created by other edges. Remove
+ // this edge from the graph. This removal happens before the edge's
+ // inputs are linked to nodes.
+ clump->edges_.erase(clump->edges_.begin() + edge_idx);
+ continue;
+ }
+ ++edge_idx;
+ }
+ }
+ }
+ {
+ // Now that all invalid edges are removed from the graph, record an out-edge
+ // on each node that's needed by an edge.
+ METRIC_RECORD(".ninja load : link edge inputs");
+ ParallelMap(thread_pool_, clumps, [](Clump* clump) {
+ for (Edge* edge : clump->edges_) {
+ for (Node* input : edge->inputs_) {
+ input->AddOutEdge(edge);
+ }
+ for (Node* validation : edge->validations_) {
+ validation->AddValidationOutEdge(edge);
+ }
+ }
+ });
+ }
+ {
+ METRIC_RECORD(".ninja load : default targets");
+
+ for (Clump* clump : clumps) {
+ for (DefaultTarget* target : clump->default_targets_) {
+ std::string path;
+ EvaluatePathInScope(&path, target->parsed_path_,
+ target->pos_.scope_pos());
+ uint64_t slash_bits; // Unused because this only does lookup.
+ std::string path_err;
+ if (!CanonicalizePath(&path, &slash_bits, &path_err))
+ return DecorateError(clump->file_, target->diag_pos_, path_err, err);
+
+ Node* node = state_->LookupNodeAtPos(path, target->pos_.dfs_location());
+ if (node == nullptr) {
+ return DecorateError(clump->file_, target->diag_pos_,
+ "unknown target '" + path + "'", err);
+ }
+ state_->AddDefault(node);
+ }
+ }
+ }
+ {
+ // Add the clump edges into the global edge vector, and assign edge IDs.
+ // Edge IDs are used for custom protobuf-based Ninja frontends. An edge's ID
+ // is equal to its index in the global edge vector, so delay the assignment
+ // of edge IDs until we've removed duplicate edges above (dupbuild=warn).
+
+ METRIC_RECORD(".ninja load : build edge table");
+
+ // Copy edges to the global edge table.
+ size_t old_size = state_->edges_.size();
+ size_t new_size = old_size;
+ for (Clump* clump : clumps) {
+ new_size += clump->edges_.size();
+ }
+ state_->edges_.reserve(new_size);
+ for (Clump* clump : clumps) {
+ std::copy(clump->edges_.begin(), clump->edges_.end(),
+ std::back_inserter(state_->edges_));
+ }
+ // Assign edge IDs.
+ ParallelMap(thread_pool_, IntegralRange<size_t>(old_size, new_size),
+ [this](size_t idx) {
+ state_->edges_[idx]->id_ = idx;
+ });
+ }
+
+ return true;
+}
+
+bool ManifestLoader::Load(ManifestFileSet* file_set,
+ const LoadedFile& root_manifest, std::string* err) {
+ DfsParser dfs_parser(file_set, state_, thread_pool_);
+ std::vector<Clump*> clumps;
+ if (!dfs_parser.LoadManifestTree(root_manifest, &state_->root_scope_, &clumps,
+ err)) {
+ return false;
+ }
+ return FinishLoading(clumps, err);
+}
+
+} // anonymous namespace
+
+bool ManifestParser::Load(const string& filename, string* err) {
+ METRIC_RECORD(".ninja load");
+
+ ManifestFileSet file_set(file_reader_);
+ const LoadedFile* file = nullptr;
+ if (!file_set.LoadFile(filename, &file, err))
+ return false;
+
+ std::unique_ptr<ThreadPool> thread_pool = CreateThreadPool();
+ ManifestLoader loader(state_, thread_pool.get(), options_, false);
+ return loader.Load(&file_set, *file, err);
+}
+
+bool ManifestParser::ParseTest(const string& input, string* err) {
+ ManifestFileSet file_set(file_reader_);
+ std::unique_ptr<ThreadPool> thread_pool = CreateThreadPool();
+ ManifestLoader loader(state_, thread_pool.get(), options_, true);
+ return loader.Load(&file_set, HeapLoadedFile("input", input), err);
+}
diff --git a/src/manifest_parser.h b/src/manifest_parser.h
index e14d069..3f1895d 100644
--- a/src/manifest_parser.h
+++ b/src/manifest_parser.h
@@ -1,4 +1,4 @@
-// Copyright 2011 Google Inc. All Rights Reserved.
+// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -15,10 +15,10 @@
#ifndef NINJA_MANIFEST_PARSER_H_
#define NINJA_MANIFEST_PARSER_H_
-#include "parser.h"
+#include <string>
-struct BindingEnv;
-struct EvalString;
+struct FileReader;
+struct State;
enum DupeEdgeAction {
kDupeEdgeActionWarn,
@@ -31,41 +31,30 @@
};
struct ManifestParserOptions {
- ManifestParserOptions()
- : dupe_edge_action_(kDupeEdgeActionWarn),
- phony_cycle_action_(kPhonyCycleActionWarn) {}
- DupeEdgeAction dupe_edge_action_;
- PhonyCycleAction phony_cycle_action_;
+ DupeEdgeAction dupe_edge_action_ = kDupeEdgeActionWarn;
+ PhonyCycleAction phony_cycle_action_ = kPhonyCycleActionWarn;
};
-/// Parses .ninja files.
-struct ManifestParser : public Parser {
+struct ManifestParser {
ManifestParser(State* state, FileReader* file_reader,
- ManifestParserOptions options = ManifestParserOptions());
+ ManifestParserOptions options = ManifestParserOptions())
+ : state_(state),
+ file_reader_(file_reader),
+ options_(options) {}
- /// Parse a text string of input. Used by tests.
- bool ParseTest(const string& input, string* err) {
- quiet_ = true;
- return Parse("input", input, err);
- }
+ /// Load and parse a file.
+ bool Load(const std::string& filename, std::string* err);
+
+ /// Parse a text string of input. Used by tests.
+ ///
+ /// Some tests may call ParseTest multiple times with the same State object.
+ /// Each call adds to the previous state; it doesn't replace it.
+ bool ParseTest(const std::string& input, std::string* err);
private:
- /// Parse a file, given its contents as a string.
- bool Parse(const string& filename, const string& input, string* err);
-
- /// Parse various statement types.
- bool ParsePool(string* err);
- bool ParseRule(string* err);
- bool ParseLet(string* key, EvalString* val, string* err);
- bool ParseEdge(string* err);
- bool ParseDefault(string* err);
-
- /// Parse either a 'subninja' or 'include' line.
- bool ParseFileInclude(bool new_scope, string* err);
-
- BindingEnv* env_;
+ State* state_ = nullptr;
+ FileReader* file_reader_ = nullptr;
ManifestParserOptions options_;
- bool quiet_;
};
#endif // NINJA_MANIFEST_PARSER_H_
diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc
index f2b7467..7a042d2 100644
--- a/src/manifest_parser_test.cc
+++ b/src/manifest_parser_test.cc
@@ -48,11 +48,10 @@
"\n"
"build result: cat in_1.cc in-2.O\n"));
- ASSERT_EQ(3u, state.bindings_.GetRules().size());
- const Rule* rule = state.bindings_.GetRules().begin()->second;
+ ASSERT_EQ(3u, state.root_scope_.GetRules().size());
+ const Rule* rule = state.root_scope_.GetRules().begin()->second;
EXPECT_EQ("cat", rule->name());
- EXPECT_EQ("[cat ][$in][ > ][$out]",
- rule->GetBinding("command")->Serialize());
+ EXPECT_EQ("cat $in > $out\n", *rule->GetBinding("command"));
}
TEST_F(ParserTest, RuleAttributes) {
@@ -63,6 +62,7 @@
" depfile = a\n"
" deps = a\n"
" description = a\n"
+" phony_output = a\n"
" generator = a\n"
" restat = a\n"
" rspfile = a\n"
@@ -81,12 +81,12 @@
"build result: cat in_1.cc in-2.O\n"
" #comment\n"));
- ASSERT_EQ(2u, state.bindings_.GetRules().size());
- const Rule* rule = state.bindings_.GetRules().begin()->second;
+ ASSERT_EQ(2u, state.root_scope_.GetRules().size());
+ const Rule* rule = state.root_scope_.GetRules().begin()->second;
EXPECT_EQ("cat", rule->name());
Edge* edge = state.GetNode("result", 0)->in_edge();
- EXPECT_TRUE(edge->GetBindingBool("restat"));
- EXPECT_FALSE(edge->GetBindingBool("generator"));
+ EXPECT_TRUE(edge->IsRestat());
+ EXPECT_FALSE(edge->IsGenerator());
}
TEST_F(ParserTest, IgnoreIndentedBlankLines) {
@@ -101,7 +101,7 @@
"variable=1\n"));
// the variable must be in the top level environment
- EXPECT_EQ("1", state.bindings_.LookupVariable("variable"));
+ EXPECT_EQ("1", state.root_scope_.LookupVariable("variable"));
}
TEST_F(ParserTest, ResponseFiles) {
@@ -114,13 +114,12 @@
"build out: cat_rsp in\n"
" rspfile=out.rsp\n"));
- ASSERT_EQ(2u, state.bindings_.GetRules().size());
- const Rule* rule = state.bindings_.GetRules().begin()->second;
+ ASSERT_EQ(2u, state.root_scope_.GetRules().size());
+ const Rule* rule = state.root_scope_.GetRules().begin()->second;
EXPECT_EQ("cat_rsp", rule->name());
- EXPECT_EQ("[cat ][$rspfile][ > ][$out]",
- rule->GetBinding("command")->Serialize());
- EXPECT_EQ("[$rspfile]", rule->GetBinding("rspfile")->Serialize());
- EXPECT_EQ("[$in]", rule->GetBinding("rspfile_content")->Serialize());
+ EXPECT_EQ("cat $rspfile > $out\n", *rule->GetBinding("command"));
+ EXPECT_EQ("$rspfile\n", *rule->GetBinding("rspfile"));
+ EXPECT_EQ("$in\n", *rule->GetBinding("rspfile_content"));
}
TEST_F(ParserTest, InNewline) {
@@ -131,11 +130,10 @@
"build out: cat_rsp in in2\n"
" rspfile=out.rsp\n"));
- ASSERT_EQ(2u, state.bindings_.GetRules().size());
- const Rule* rule = state.bindings_.GetRules().begin()->second;
+ ASSERT_EQ(2u, state.root_scope_.GetRules().size());
+ const Rule* rule = state.root_scope_.GetRules().begin()->second;
EXPECT_EQ("cat_rsp", rule->name());
- EXPECT_EQ("[cat ][$in_newline][ > ][$out]",
- rule->GetBinding("command")->Serialize());
+ EXPECT_EQ("cat $in_newline > $out\n", *rule->GetBinding("command"));
Edge* edge = state.edges_[0];
EXPECT_EQ("cat in\nin2 > out", edge->EvaluateCommand());
@@ -159,7 +157,7 @@
Edge* edge = state.edges_[0];
EXPECT_EQ("ld one-letter-test -pthread -under -o a b c",
edge->EvaluateCommand());
- EXPECT_EQ("1/2", state.bindings_.LookupVariable("nested2"));
+ EXPECT_EQ("1/2", state.root_scope_.LookupVariable("nested2"));
edge = state.edges_[1];
EXPECT_EQ("ld one-letter-test 1/2/3 -under -o supernested x",
@@ -192,10 +190,12 @@
"build a: link c $\n"
" d e f\n"));
- ASSERT_EQ(2u, state.bindings_.GetRules().size());
- const Rule* rule = state.bindings_.GetRules().begin()->second;
+ ASSERT_EQ(2u, state.root_scope_.GetRules().size());
+ const Rule* rule = state.root_scope_.GetRules().begin()->second;
EXPECT_EQ("link", rule->name());
- EXPECT_EQ("[foo bar baz]", rule->GetBinding("command")->Serialize());
+ std::string command = *rule->GetBinding("command");
+ EXPECT_EQ("foo bar $\n baz\n", command);
+ EXPECT_EQ("foo bar baz", EvaluateBindingForTesting(command));
}
TEST_F(ParserTest, Backslash) {
@@ -203,15 +203,15 @@
"foo = bar\\baz\n"
"foo2 = bar\\ baz\n"
));
- EXPECT_EQ("bar\\baz", state.bindings_.LookupVariable("foo"));
- EXPECT_EQ("bar\\ baz", state.bindings_.LookupVariable("foo2"));
+ EXPECT_EQ("bar\\baz", state.root_scope_.LookupVariable("foo"));
+ EXPECT_EQ("bar\\ baz", state.root_scope_.LookupVariable("foo2"));
}
TEST_F(ParserTest, Comment) {
ASSERT_NO_FATAL_FAILURE(AssertParse(
"# this is a comment\n"
"foo = not # a comment\n"));
- EXPECT_EQ("not # a comment", state.bindings_.LookupVariable("foo"));
+ EXPECT_EQ("not # a comment", state.root_scope_.LookupVariable("foo"));
}
TEST_F(ParserTest, Dollars) {
@@ -222,7 +222,7 @@
"x = $$dollar\n"
"build $x: foo y\n"
));
- EXPECT_EQ("$dollar", state.bindings_.LookupVariable("x"));
+ EXPECT_EQ("$dollar", state.root_scope_.LookupVariable("x"));
#ifdef _WIN32
EXPECT_EQ("$dollarbar$baz$blah", state.edges_[0]->EvaluateCommand());
#else
@@ -418,6 +418,38 @@
"default subninja\n"));
}
+TEST_F(ParserTest, NulCharErrors) {
+ {
+ State local_state;
+ ManifestParser parser(&local_state, NULL);
+ std::string err;
+ EXPECT_FALSE(parser.ParseTest("\0"_s, &err));
+ EXPECT_EQ("input:1: unexpected NUL byte\n", err);
+ }
+
+ {
+ State local_state;
+ ManifestParser parser(&local_state, NULL);
+ std::string err;
+ EXPECT_FALSE(parser.ParseTest("build foo\0"_s, &err));
+ EXPECT_EQ("input:1: unexpected NUL byte\n"
+ "build foo\n"
+ " ^ near here"
+ , err);
+ }
+
+ {
+ State local_state;
+ ManifestParser parser(&local_state, NULL);
+ std::string err;
+ EXPECT_FALSE(parser.ParseTest("foo\0"_s, &err));
+ EXPECT_EQ("input:1: expected '=', got nul byte\n"
+ "foo\n"
+ " ^ near here"
+ , err);
+ }
+}
+
TEST_F(ParserTest, Errors) {
{
State local_state;
@@ -707,10 +739,15 @@
}
{
+ // The default statement's target must be listed earlier in a build
+ // statement.
State local_state;
ManifestParser parser(&local_state, NULL);
string err;
- EXPECT_FALSE(parser.ParseTest("default nonexistent\n",
+ EXPECT_FALSE(parser.ParseTest("default nonexistent\n"
+ "rule cat\n"
+ " command = cat $in > $out\n"
+ "build nonexistent: cat existent\n",
&err));
EXPECT_EQ("input:1: unknown target 'nonexistent'\n"
"default nonexistent\n"
@@ -792,7 +829,8 @@
string err;
EXPECT_FALSE(parser.ParseTest("pool foo\n"
" depth = 4\n"
- "pool foo\n", &err));
+ "pool foo\n"
+ " depth = 2\n", &err));
EXPECT_EQ("input:3: duplicate pool 'foo'\n"
"pool foo\n"
" ^ near here"
@@ -854,6 +892,16 @@
EXPECT_EQ("", err);
}
+TEST_F(ParserTest, MultipleImplicitOutputsWithDeps) {
+ State local_state;
+ ManifestParser parser(&local_state, NULL);
+ string err;
+ EXPECT_TRUE(parser.ParseTest("rule cc\n command = foo\n deps = gcc\n"
+ "build a.o | a.gcno: cc c.cc\n",
+ &err));
+ EXPECT_EQ("", err);
+}
+
TEST_F(ParserTest, MultipleOutputsWithDeps) {
State local_state;
ManifestParser parser(&local_state, NULL);
@@ -932,7 +980,7 @@
ASSERT_EQ(1u, fs_.files_read_.size());
EXPECT_EQ("include.ninja", fs_.files_read_[0]);
- EXPECT_EQ("inner", state.bindings_.LookupVariable("var"));
+ EXPECT_EQ("inner", state.root_scope_.LookupVariable("var"));
}
TEST_F(ParserTest, BrokenInclude) {
@@ -965,6 +1013,16 @@
ASSERT_TRUE(edge->is_order_only(1));
}
+TEST_F(ParserTest, Validations) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(
+"rule cat\n command = cat $in > $out\n"
+"build foo: cat bar |@ baz\n"));
+
+ Edge* edge = state.LookupNode("foo")->in_edge();
+ ASSERT_EQ(edge->validations_.size(), 1);
+ EXPECT_EQ(edge->validations_[0]->path(), "baz");
+}
+
TEST_F(ParserTest, ImplicitOutput) {
ASSERT_NO_FATAL_FAILURE(AssertParse(
"rule cat\n"
@@ -1155,3 +1213,67 @@
EXPECT_TRUE(edge->dyndep_->dyndep_pending());
EXPECT_EQ(edge->dyndep_->path(), "in");
}
+
+TEST_F(ParserTest, IncludeUsingAVariable) {
+ // Each include statement should use the $path binding from the previous
+ // manifest declarations.
+ fs_.Create("foo.ninja", "path = bar.ninja\n"
+ "include $path\n");
+ fs_.Create("bar.ninja", "");
+ ASSERT_NO_FATAL_FAILURE(AssertParse(
+"path = foo.ninja\n"
+"include $path\n"
+"include $path\n"
+"path = nonexistent.ninja\n"));
+}
+
+TEST_F(ParserTest, UnscopedPool) {
+ // Pools aren't scoped.
+ fs_.Create("foo.ninja", "pool link\n"
+ " depth = 3\n");
+ ASSERT_NO_FATAL_FAILURE(AssertParse(
+"rule cat\n"
+" command = cat $in > $out\n"
+" pool = link\n"
+"subninja foo.ninja\n"
+"build a: cat b\n"));
+}
+
+TEST_F(ParserTest, PoolDeclaredAfterUse) {
+ // A pool must be declared before an edge that uses it.
+ ManifestParser parser(&state, nullptr);
+ std::string err;
+ EXPECT_FALSE(parser.ParseTest("rule cat\n"
+ " command = cat $in > $out\n"
+ " pool = link\n"
+ "build a: cat b\n"
+ "pool link\n"
+ " depth = 3\n", &err));
+ EXPECT_EQ("input:5: unknown pool name 'link'\n", err);
+}
+
+TEST_F(ParserTest, DefaultReferencesEdgeInput) {
+ // The default statement's target must be listed before the default statement.
+ // It's OK if the target's first reference is an input, though.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(
+"rule cat\n command = cat $in > $out\n"
+"build a1: cat b1\n"
+"build a2: cat b2\n"
+"default b1 b2\n"
+"build b1: cat c1\n"));
+}
+
+TEST_F(ParserTest, SelfVarExpansion) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(
+"var = xxx\n"
+"var1 = ${var}\n"
+"var = a${var}a\n"
+"var2 = ${var}\n"
+"var = b${var}b\n"
+"var3 = ${var}\n"
+"var = c${var}c\n"));
+ EXPECT_EQ("xxx", state.root_scope_.LookupVariable("var1"));
+ EXPECT_EQ("axxxa", state.root_scope_.LookupVariable("var2"));
+ EXPECT_EQ("baxxxab", state.root_scope_.LookupVariable("var3"));
+ EXPECT_EQ("cbaxxxabc", state.root_scope_.LookupVariable("var"));
+}
diff --git a/src/metrics.cc b/src/metrics.cc
index a7d3c7a..5e4cef9 100644
--- a/src/metrics.cc
+++ b/src/metrics.cc
@@ -19,13 +19,17 @@
#include <string.h>
#ifndef _WIN32
+#include <sys/resource.h>
#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
#else
#include <windows.h>
#endif
#include <algorithm>
+#include "status.h"
#include "util.h"
Metrics* g_metrics = NULL;
@@ -75,45 +79,40 @@
} // anonymous namespace
-ScopedMetric::ScopedMetric(Metric* metric) {
- metric_ = metric;
- if (!metric_)
- return;
+void ScopedMetric::RecordStart() {
start_ = HighResTimer();
}
-ScopedMetric::~ScopedMetric() {
- if (!metric_)
- return;
- metric_->count++;
- int64_t dt = TimerToMicros(HighResTimer() - start_);
- metric_->sum += dt;
+
+void ScopedMetric::RecordResult() {
+ int64_t duration = TimerToMicros(HighResTimer() - start_);
+ metric_->AddResult(1, duration);
}
Metric* Metrics::NewMetric(const string& name) {
- Metric* metric = new Metric;
- metric->name = name;
- metric->count = 0;
- metric->sum = 0;
- metrics_.push_back(metric);
- return metric;
+ std::lock_guard<std::mutex> lock(mutex_);
+ Metric* result = new Metric(name);
+ metrics_.push_back(result);
+ return result;
}
-void Metrics::Report() {
+void Metrics::Report(Status *status) {
+ std::lock_guard<std::mutex> lock(mutex_);
+
int width = 0;
for (vector<Metric*>::iterator i = metrics_.begin();
i != metrics_.end(); ++i) {
- width = max((int)(*i)->name.size(), width);
+ width = max((int)(*i)->name().size(), width);
}
- printf("%-*s\t%-6s\t%-9s\t%s\n", width,
+ status->Debug("%-*s\t%-6s\t%-9s\t%s", width,
"metric", "count", "avg (us)", "total (ms)");
for (vector<Metric*>::iterator i = metrics_.begin();
i != metrics_.end(); ++i) {
Metric* metric = *i;
- double total = metric->sum / (double)1000;
- double avg = metric->sum / (double)metric->count;
- printf("%-*s\t%-6d\t%-8.1f\t%.1f\n", width, metric->name.c_str(),
- metric->count, avg, total);
+ double total = metric->time() / (double)1000;
+ double avg = metric->time() / (double)metric->count();
+ status->Debug("%-*s\t%-6d\t%-8.1f\t%.1f", width, metric->name().c_str(),
+ metric->count(), avg, total);
}
}
@@ -125,3 +124,39 @@
return TimerToMicros(HighResTimer()) / 1000;
}
+void DumpMemoryUsage(Status *status) {
+#if defined(__linux__)
+ std::vector<std::string> words;
+ struct rusage usage {};
+ if (getrusage(RUSAGE_SELF, &usage) == 0) {
+ words.push_back(std::to_string(usage.ru_majflt) + " maj faults");
+ words.push_back(std::to_string(usage.ru_minflt) + " min faults");
+ words.push_back(std::to_string(usage.ru_maxrss / 1024) + " MiB maxrss");
+ }
+ char status_path[256];
+ snprintf(status_path, sizeof(status_path), "/proc/%d/status",
+ static_cast<int>(getpid()));
+ if (FILE* status_fp = fopen(status_path, "r")) {
+ char* line = nullptr;
+ size_t n = 0;
+ while (getline(&line, &n, status_fp) > 0) {
+ unsigned long rss = 0;
+ if (sscanf(line, "VmRSS:\t%lu kB\n", &rss) == 1) {
+ words.push_back(std::to_string(rss / 1024) + " MiB rss");
+ }
+ }
+ free(line);
+ fclose(status_fp);
+ }
+ if (!words.empty()) {
+ string final;
+ for (size_t i = 0; i < words.size(); ++i) {
+ if (i > 0) {
+ final += ", ";
+ }
+ final += words[i];
+ }
+ status->Debug("%s", final.c_str());
+ }
+#endif
+}
diff --git a/src/metrics.h b/src/metrics.h
index b6da859..48f26d1 100644
--- a/src/metrics.h
+++ b/src/metrics.h
@@ -15,32 +15,83 @@
#ifndef NINJA_METRICS_H_
#define NINJA_METRICS_H_
+#include <atomic>
+#include <mutex>
#include <string>
#include <vector>
using namespace std;
#include "util.h" // For int64_t.
+struct Status;
+
/// The Metrics module is used for the debug mode that dumps timing stats of
/// various actions. To use, see METRIC_RECORD below.
/// A single metrics we're tracking, like "depfile load time".
struct Metric {
- string name;
- /// Number of times we've hit the code path.
- int count;
- /// Total time (in micros) we've spent on the code path.
- int64_t sum;
-};
+ Metric(const std::string& name) : name_(name) {}
+ void AddResult(int count, int64_t time_us) {
+ thread_local size_t this_slot = GetThreadSlotIndex();
+ Slot& slot = slots_[this_slot];
+ slot.count += count;
+ slot.time += time_us;
+ }
-/// A scoped object for recording a metric across the body of a function.
-/// Used by the METRIC_RECORD macro.
-struct ScopedMetric {
- explicit ScopedMetric(Metric* metric);
- ~ScopedMetric();
+ const std::string& name() const { return name_; }
+
+ int count() const {
+ int result = 0;
+ for (const Slot& slot : slots_) {
+ result += slot.count;
+ }
+ return result;
+ }
+
+ int64_t time() const {
+ int64_t result = 0;
+ for (const Slot& slot : slots_) {
+ result += slot.time;
+ }
+ return result;
+ }
private:
+ /// Try to give each thread a different slot to reduce thread contention.
+ struct NINJA_ALIGNAS_CACHE_LINE Slot {
+ /// Number of times we've hit the code path.
+ std::atomic<int> count {};
+ /// Total time (in micros) we've spent on the code path.
+ std::atomic<int64_t> time {};
+ };
+
+ std::string name_;
+ std::vector<Slot> slots_ { GetThreadSlotCount() };
+};
+
+/// A scoped object for recording a metric across the body of a function.
+/// Used by the METRIC_RECORD macro. Inline the metric_ null check to minimize
+/// the effect on the typical case where metrics are disabled.
+struct ScopedMetric {
+ explicit ScopedMetric(Metric* metric) : metric_(metric) {
+ if (metric_ == nullptr) {
+ return;
+ }
+ RecordStart();
+ }
+
+ ~ScopedMetric() {
+ if (metric_ == nullptr) {
+ return;
+ }
+ RecordResult();
+ }
+
+private:
+ void RecordStart();
+ void RecordResult();
+
Metric* metric_;
/// Timestamp when the measurement started.
/// Value is platform-dependent.
@@ -52,9 +103,10 @@
Metric* NewMetric(const string& name);
/// Print a summary report to stdout.
- void Report();
+ void Report(Status *status);
private:
+ std::mutex mutex_;
vector<Metric*> metrics_;
};
@@ -89,4 +141,6 @@
extern Metrics* g_metrics;
+void DumpMemoryUsage(Status *status);
+
#endif // NINJA_METRICS_H_
diff --git a/src/ninja.cc b/src/ninja.cc
index 5f19a65..65cc8aa 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -30,6 +30,9 @@
#include <unistd.h>
#endif
+#include <deque>
+#include <unordered_map>
+
#include "browse.h"
#include "build.h"
#include "build_log.h"
@@ -42,6 +45,8 @@
#include "manifest_parser.h"
#include "metrics.h"
#include "state.h"
+#include "status.h"
+#include "thread_pool.h"
#include "util.h"
#include "version.h"
@@ -53,6 +58,11 @@
void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep);
#endif
+// Ninja intentionally leaks memory. Turn off LeakSanitizer by default.
+extern "C" const char* __asan_default_options() {
+ return "detect_leaks=0";
+}
+
namespace {
struct Tool;
@@ -77,13 +87,17 @@
/// Whether a depfile with multiple targets on separate lines should
/// warn or print an error.
bool depfile_distinct_target_lines_should_err;
+
+ /// Whether to remain persistent.
+ bool persistent;
};
/// The Ninja main() loads up a series of data structures; various tools need
/// to poke into these, so store them as fields on an object.
struct NinjaMain : public BuildLogUser {
NinjaMain(const char* ninja_command, const BuildConfig& config) :
- ninja_command_(ninja_command), config_(config) {}
+ ninja_command_(ninja_command), config_(config),
+ start_time_millis_(GetTimeMillis()) {}
/// Command line used to run Ninja.
const char* ninja_command_;
@@ -116,6 +130,8 @@
// The various subcommands, run via "-t XXX".
int ToolGraph(const Options* options, int argc, char* argv[]);
+ int ToolPath(const Options* options, int argc, char* argv[]);
+ int ToolInputs(const Options* options, int argc, char* argv[]);
int ToolQuery(const Options* options, int argc, char* argv[]);
int ToolDeps(const Options* options, int argc, char* argv[]);
int ToolBrowse(const Options* options, int argc, char* argv[]);
@@ -142,14 +158,14 @@
/// Rebuild the manifest, if necessary.
/// Fills in \a err on error.
/// @return true if the manifest was rebuilt.
- bool RebuildManifest(const char* input_file, string* err);
+ bool RebuildManifest(const char* input_file, string* err, Status* status);
/// Build the targets listed on the command line.
/// @return an exit code.
- int RunBuild(int argc, char** argv);
+ int RunBuild(int argc, char** argv, Status* status);
/// Dump the output requested by '-d stats'.
- void DumpMetrics();
+ void DumpMetrics(Status *status);
virtual bool IsPathDead(StringPiece s) const {
Node* n = state_.LookupNode(s);
@@ -165,11 +181,13 @@
// Do keep entries around for files which still exist on disk, for
// generators that want to use this information.
string err;
- TimeStamp mtime = disk_interface_.Stat(s.AsString(), &err);
+ TimeStamp mtime = disk_interface_.LStat(s.AsString(), nullptr, &err);
if (mtime == -1)
Error("%s", err.c_str()); // Log and ignore Stat() errors.
return mtime == 0;
}
+
+ int64_t start_time_millis_;
};
/// Subtools, accessible via "-t foo".
@@ -219,8 +237,14 @@
" -d MODE enable debugging (use '-d list' to list modes)\n"
" -t TOOL run a subtool (use '-t list' to list subtools)\n"
" terminates toplevel options; further flags are passed to the tool\n"
-" -w FLAG adjust warnings (use '-w list' to list warnings)\n",
- kNinjaVersion, config.parallelism);
+" -o FLAG adjust options (use '-o list' to list options)\n"
+" -w FLAG adjust warnings (use '-w list' to list warnings)\n"
+#ifndef _WIN32
+"\n"
+" --frontend COMMAND execute COMMAND and pass serialized build output to it\n"
+" --frontend_file FILE write serialized build output to FILE\n"
+#endif
+ , kNinjaVersion, config.parallelism);
}
/// Choose a default value for the -j (parallelism) flag.
@@ -238,7 +262,8 @@
/// Rebuild the build manifest, if necessary.
/// Returns true if the manifest was rebuilt.
-bool NinjaMain::RebuildManifest(const char* input_file, string* err) {
+bool NinjaMain::RebuildManifest(const char* input_file, string* err,
+ Status* status) {
string path = input_file;
uint64_t slash_bits; // Unused because this path is only used for lookup.
if (!CanonicalizePath(&path, &slash_bits, err))
@@ -247,8 +272,9 @@
if (!node)
return false;
- Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
- if (!builder.AddTarget(node, err))
+ Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_,
+ status, start_time_millis_);
+ if (!builder.AddTargets({ node }, err))
return false;
if (builder.AlreadyUpToDate())
@@ -285,11 +311,11 @@
Node* node = state_.LookupNode(path);
if (node) {
if (first_dependent) {
- if (node->out_edges().empty()) {
+ if (!node->has_out_edge()) {
*err = "'" + path + "' has no out edge";
return NULL;
}
- Edge* edge = node->out_edges()[0];
+ Edge* edge = node->GetOutEdges()[0];
if (edge->outputs_.empty()) {
edge->Dump();
Fatal("edge has no outputs");
@@ -347,6 +373,156 @@
return 0;
}
+int NinjaMain::ToolPath(const Options* options, int argc, char* argv[]) {
+ if (argc != 2) {
+ Error("expected two targets to find a dependency chain between");
+ return 1;
+ }
+ std::string err;
+ Node* out = CollectTarget(argv[0], &err);
+ if (!out) {
+ Error("%s", err.c_str());
+ return 1;
+ }
+ Node* in = CollectTarget(argv[1], &err);
+ if (!in) {
+ Error("%s", err.c_str());
+ return 1;
+ }
+
+ // BFS from "in", looking for "out". The map record a node's input that leads
+ // back to "in".
+ std::unordered_map<Node*, Node*> node_next;
+ std::deque<Node*> queue;
+ queue.push_back(in);
+ while (!queue.empty()) {
+ Node* node = queue[0];
+ queue.pop_front();
+ if (node == out) {
+ for (; node != nullptr; node = node_next[node]) {
+ printf("%s\n", node->path().c_str());
+ }
+ return 0;
+ }
+ for (Edge* edge : node->GetOutEdges()) {
+ for (Node* output : edge->outputs_) {
+ if (node_next.count(output) == 0) {
+ node_next[output] = node;
+ queue.push_back(output);
+ }
+ }
+ }
+ for (Edge* edge : node->GetValidationOutEdges()) {
+ for (Node* output : edge->outputs_) {
+ if (node_next.count(output) == 0) {
+ node_next[output] = node;
+ queue.push_back(output);
+ }
+ }
+ }
+ }
+ Error("%s does not depend on %s", out->path().c_str(), in->path().c_str());
+ return 1;
+}
+
+void ToolInputsProcessNodeDeps(Node* node, DepsLog* deps_log, bool leaf_only,
+ bool include_deps);
+
+void ToolInputsProcessNode(Node* input, DepsLog* deps_log, bool leaf_only,
+ bool include_deps) {
+ if (input->InputsChecked()) return;
+ input->MarkInputsChecked();
+
+ // Recursively process input edges, possibly printing this node here.
+ Edge* input_edge = input->in_edge();
+ if (input_edge == nullptr || !leaf_only) {
+ printf("%s\n", input->path().c_str());
+ }
+ if (input_edge) {
+ for (Node* input : input_edge->inputs_) {
+ ToolInputsProcessNode(input, deps_log, leaf_only, include_deps);
+ }
+ if (include_deps && input_edge->outputs_.size() > 0) {
+ // Check deps on the input edge's first output because deps log entries
+ // are only stored on the first output of an edge.
+ ToolInputsProcessNodeDeps(input_edge->outputs_[0], deps_log, leaf_only,
+ include_deps);
+ }
+ }
+}
+
+void ToolInputsProcessNodeDeps(Node* node, DepsLog* deps_log, bool leaf_only,
+ bool include_deps) {
+ // Print all of this node's deps from the deps log. This often includes files
+ // that are not known by the node's input edge.
+ if (deps_log->IsDepsEntryLiveFor(node)) {
+ if (DepsLog::Deps* deps = deps_log->GetDeps(node); deps != nullptr) {
+ for (int i = 0; i < deps->node_count; ++i) {
+ if (Node* dep = deps->nodes[i]; dep != nullptr) {
+ ToolInputsProcessNode(dep, deps_log, leaf_only, include_deps);
+ }
+ }
+ }
+ }
+}
+
+int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) {
+ // The inputs tool uses getopt, and expects argv[0] to contain the name of
+ // the tool, i.e. "inputs".
+ ++argc;
+ --argv;
+
+ bool leaf_only = true;
+ bool include_deps = false;
+
+ optind = 1;
+ int opt;
+ while ((opt = getopt(argc, argv, "idh")) != -1) {
+ switch (opt) {
+ case 'i':
+ leaf_only = false;
+ break;
+ case 'd':
+ include_deps = true;
+ break;
+ case 'h':
+ default:
+ printf(
+ "usage: ninja -t inputs [options] target [target...]\n\n"
+ "options:\n"
+ " -i Include intermediate inputs.\n"
+ " -d Include deps from the deps log file (.ninja_deps).\n");
+ return 1;
+ }
+ }
+ argv += optind;
+ argc -= optind;
+
+ vector<Node*> nodes;
+ string err;
+ if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) {
+ Error("%s", err.c_str());
+ return 1;
+ }
+
+ for (Node* node : nodes) {
+ node->MarkInputsChecked();
+ // Call ToolInputsProcessNode on this node's inputs, and not on itself,
+ // so that this node is not included in the output.
+ if (Edge* edge = node->in_edge(); edge != nullptr) {
+ for (Node* input : edge->inputs_) {
+ ToolInputsProcessNode(input, &deps_log_, leaf_only, include_deps);
+ }
+ if (include_deps && edge->outputs_.size() > 0) {
+ ToolInputsProcessNodeDeps(edge->outputs_[0], &deps_log_, leaf_only,
+ include_deps);
+ }
+ }
+ }
+ return 0;
+}
+
+
int NinjaMain::ToolQuery(const Options* options, int argc, char* argv[]) {
if (argc == 0) {
Error("expected a target to query");
@@ -379,15 +555,33 @@
label = "|| ";
printf(" %s%s\n", label, edge->inputs_[in]->path().c_str());
}
+ if (!edge->validations_.empty()) {
+ printf(" validations:\n");
+ for (Node* validation : edge->validations_) {
+ printf(" %s\n", validation->path().c_str());
+ }
+ }
}
printf(" outputs:\n");
- for (vector<Edge*>::const_iterator edge = node->out_edges().begin();
- edge != node->out_edges().end(); ++edge) {
+ const std::vector<Edge*> out_edges = node->GetOutEdges();
+ for (vector<Edge*>::const_iterator edge = out_edges.begin();
+ edge != out_edges.end(); ++edge) {
for (vector<Node*>::iterator out = (*edge)->outputs_.begin();
out != (*edge)->outputs_.end(); ++out) {
printf(" %s\n", (*out)->path().c_str());
}
}
+ const std::vector<Edge*> validation_edges = node->GetValidationOutEdges();
+ if (!validation_edges.empty()) {
+ printf(" validation for:\n");
+ for (vector<Edge*>::const_iterator edge = validation_edges.begin();
+ edge != validation_edges.end(); ++edge) {
+ for (vector<Node*>::iterator out = (*edge)->outputs_.begin();
+ out != (*edge)->outputs_.end(); ++out) {
+ printf(" %s\n", (*out)->path().c_str());
+ }
+ }
+ }
}
return 0;
}
@@ -562,7 +756,7 @@
}
enum PrintCommandMode { PCM_Single, PCM_All };
-void PrintCommands(Edge* edge, set<Edge*>* seen, PrintCommandMode mode) {
+void PrintCommands(Edge* edge, EdgeSet* seen, PrintCommandMode mode) {
if (!edge)
return;
if (!seen->insert(edge).second)
@@ -613,7 +807,7 @@
return 1;
}
- set<Edge*> seen;
+ EdgeSet seen;
for (vector<Node*>::iterator in = nodes.begin(); in != nodes.end(); ++in)
PrintCommands((*in)->in_edge(), &seen, mode);
@@ -833,6 +1027,10 @@
Tool::RUN_AFTER_LOGS, &NinjaMain::ToolDeps },
{ "graph", "output graphviz dot file for targets",
Tool::RUN_AFTER_LOAD, &NinjaMain::ToolGraph },
+ { "inputs", "show all (recursive) inputs for a target",
+ Tool::RUN_AFTER_LOGS, &NinjaMain::ToolInputs },
+ { "path", "find dependency path between two targets",
+ Tool::RUN_AFTER_LOGS, &NinjaMain::ToolPath },
{ "query", "show inputs/outputs for a path",
Tool::RUN_AFTER_LOGS, &NinjaMain::ToolQuery },
{ "targets", "list targets by their rule or depth in the DAG",
@@ -885,6 +1083,7 @@
#ifdef _WIN32
" nostatcache don't batch stat() calls per directory and cache them\n"
#endif
+" nothreads don't use threads to parallelize ninja\n"
"multiple modes can be enabled via -d FOO -d BAR\n");
return false;
} else if (name == "stats") {
@@ -902,11 +1101,14 @@
} else if (name == "nostatcache") {
g_experimental_statcache = false;
return true;
+ } else if (name == "nothreads") {
+ g_use_threads = false;
+ return true;
} else {
const char* suggestion =
SpellcheckString(name.c_str(),
"stats", "explain", "keepdepfile", "keeprsp",
- "nostatcache", NULL);
+ "nostatcache", "nothreads", NULL);
if (suggestion) {
Error("unknown debug setting '%s', did you mean '%s'?",
name.c_str(), suggestion);
@@ -919,13 +1121,18 @@
/// Set a warning flag. Returns false if Ninja should exit instead of
/// continuing.
-bool WarningEnable(const string& name, Options* options) {
+bool WarningEnable(const string& name, Options* options, BuildConfig* config) {
if (name == "list") {
printf("warning flags:\n"
" dupbuild={err,warn} multiple build lines for one target\n"
" phonycycle={err,warn} phony build statement references itself\n"
" depfilemulti={err,warn} depfile has multiple output paths on separate lines\n"
- );
+" missingdepfile={err,warn} how to treat missing depfiles\n"
+"\n"
+" requires -o usesphonyoutputs=yes\n"
+" outputdir={err,warn} how to treat outputs that are directories\n"
+" missingoutfile={err,warn} how to treat missing output files\n"
+" oldoutput={err,warn} how to treat output files older than their inputs\n");
return false;
} else if (name == "dupbuild=err") {
options->dupe_edges_should_err = true;
@@ -945,10 +1152,38 @@
} else if (name == "depfilemulti=warn") {
options->depfile_distinct_target_lines_should_err = false;
return true;
+ } else if (name == "missingdepfile=err") {
+ config->missing_depfile_should_err = true;
+ return true;
+ } else if (name == "missingdepfile=warn") {
+ config->missing_depfile_should_err = false;
+ return true;
+ } else if (name == "outputdir=err") {
+ config->output_directory_should_err = true;
+ return true;
+ } else if (name == "outputdir=warn") {
+ config->output_directory_should_err = false;
+ return true;
+ } else if (name == "missingoutfile=err") {
+ config->missing_output_file_should_err = true;
+ return true;
+ } else if (name == "missingoutfile=warn") {
+ config->missing_output_file_should_err = false;
+ return true;
+ } else if (name == "oldoutput=err") {
+ config->old_output_should_err = true;
+ return true;
+ } else if (name == "oldoutput=warn") {
+ config->old_output_should_err = false;
+ return true;
} else {
const char* suggestion =
SpellcheckString(name.c_str(), "dupbuild=err", "dupbuild=warn",
- "phonycycle=err", "phonycycle=warn", NULL);
+ "phonycycle=err", "phonycycle=warn",
+ "missingdepfile=err", "missingdepfile=warn",
+ "outputdir=err", "outputdir=warn",
+ "missingoutfile=err", "missingoutfile=warn",
+ "oldoutput=err", "oldoutput=warn", NULL);
if (suggestion) {
Error("unknown warning flag '%s', did you mean '%s'?",
name.c_str(), suggestion);
@@ -959,6 +1194,46 @@
}
}
+/// Set an option flag. Returns false if Ninja should exit instead of
+/// continuing.
+bool OptionEnable(const string& name, Options* options, BuildConfig* config) {
+ if (name == "list") {
+ printf("option flags:\n"
+" usesphonyoutputs={yes,no} whether the generate uses 'phony_output's so \n"
+" that these warnings work:\n"
+" outputdir\n"
+" missingoutfile\n"
+" oldoutput\n"
+" preremoveoutputs={yes,no} whether to remove outputs before running rule\n");
+ return false;
+ } else if (name == "usesphonyoutputs=yes") {
+ config->uses_phony_outputs = true;
+ return true;
+ } else if (name == "usesphonyoutputs=no") {
+ config->uses_phony_outputs = false;
+ return true;
+ } else if (name == "preremoveoutputs=yes") {
+ config->pre_remove_output_files = true;
+ return true;
+ } else if (name == "preremoveoutputs=no") {
+ config->pre_remove_output_files = false;
+ return true;
+ } else {
+ const char* suggestion =
+ SpellcheckString(name.c_str(),
+ "usesphonyoutputs=yes", "usesphonyoutputs=no",
+ "preremoveoutputs=yes", "preremoveoutputs=no",
+ NULL);
+ if (suggestion) {
+ Error("unknown option flag '%s', did you mean '%s'?",
+ name.c_str(), suggestion);
+ } else {
+ Error("unknown option flag '%s'", name.c_str());
+ }
+ return false;
+ }
+}
+
bool NinjaMain::OpenBuildLog(bool recompact_only) {
string log_path = ".ninja_log";
if (!build_dir_.empty())
@@ -1011,14 +1286,14 @@
}
if (recompact_only) {
- bool success = deps_log_.Recompact(path, &err);
+ bool success = deps_log_.Recompact(path, disk_interface_, &err);
if (!success)
Error("failed recompaction: %s", err.c_str());
return success;
}
if (!config_.dry_run) {
- if (!deps_log_.OpenForWrite(path, &err)) {
+ if (!deps_log_.OpenForWrite(path, disk_interface_, &err)) {
Error("opening deps log: %s", err.c_str());
return false;
}
@@ -1027,18 +1302,20 @@
return true;
}
-void NinjaMain::DumpMetrics() {
- g_metrics->Report();
+void NinjaMain::DumpMetrics(Status *status) {
+ g_metrics->Report(status);
- printf("\n");
+ status->Debug("");
+
int count = (int)state_.paths_.size();
int buckets = (int)state_.paths_.bucket_count();
- printf("path->node hash load %.2f (%d entries / %d buckets)\n",
- count / (double) buckets, count, buckets);
+ status->Debug("path->node hash load %.2f (%d entries / %d buckets), %zu edges",
+ count / (double) buckets, count, buckets, state_.edges_.size());
+ DumpMemoryUsage(status);
}
bool NinjaMain::EnsureBuildDirExists() {
- build_dir_ = state_.bindings_.LookupVariable("builddir");
+ build_dir_ = state_.root_scope_.LookupVariable("builddir");
if (!build_dir_.empty() && !config_.dry_run) {
if (!disk_interface_.MakeDirs(build_dir_ + "/.") && errno != EEXIST) {
Error("creating build directory %s: %s",
@@ -1049,39 +1326,33 @@
return true;
}
-int NinjaMain::RunBuild(int argc, char** argv) {
+int NinjaMain::RunBuild(int argc, char** argv, Status* status) {
string err;
vector<Node*> targets;
if (!CollectTargetsFromArgs(argc, argv, &targets, &err)) {
- Error("%s", err.c_str());
+ status->Error("%s", err.c_str());
return 1;
}
disk_interface_.AllowStatCache(g_experimental_statcache);
- Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
- for (size_t i = 0; i < targets.size(); ++i) {
- if (!builder.AddTarget(targets[i], &err)) {
- if (!err.empty()) {
- Error("%s", err.c_str());
- return 1;
- } else {
- // Added a target that is already up-to-date; not really
- // an error.
- }
- }
+ Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_,
+ status, start_time_millis_);
+ if (!builder.AddTargets(targets, &err)) {
+ status->Error("%s", err.c_str());
+ return 1;
}
// Make sure restat rules do not see stale timestamps.
disk_interface_.AllowStatCache(false);
if (builder.AlreadyUpToDate()) {
- printf("ninja: no work to do.\n");
+ status->Info("no work to do.");
return 0;
}
if (!builder.Build(&err)) {
- printf("ninja: build stopped: %s.\n", err.c_str());
+ status->Info("build stopped: %s.", err.c_str());
if (err.find("interrupted by user") != string::npos) {
return 2;
}
@@ -1120,8 +1391,16 @@
Options* options, BuildConfig* config) {
config->parallelism = GuessParallelism();
- enum { OPT_VERSION = 1 };
+ enum {
+ OPT_VERSION = 1,
+ OPT_FRONTEND = 2,
+ OPT_FRONTEND_FILE = 3,
+ };
const option kLongOptions[] = {
+#ifndef _WIN32
+ { "frontend", required_argument, NULL, OPT_FRONTEND },
+ { "frontend_file", required_argument, NULL, OPT_FRONTEND_FILE },
+#endif
{ "help", no_argument, NULL, 'h' },
{ "version", no_argument, NULL, OPT_VERSION },
{ "verbose", no_argument, NULL, 'v' },
@@ -1130,7 +1409,7 @@
int opt;
while (!options->tool &&
- (opt = getopt_long(*argc, *argv, "d:f:j:k:l:nt:vw:C:h", kLongOptions,
+ (opt = getopt_long(*argc, *argv, "d:f:j:k:l:mnt:vw:o:C:ph", kLongOptions,
NULL)) != -1) {
switch (opt) {
case 'd':
@@ -1183,15 +1462,28 @@
config->verbosity = BuildConfig::VERBOSE;
break;
case 'w':
- if (!WarningEnable(optarg, options))
+ if (!WarningEnable(optarg, options, config))
+ return 1;
+ break;
+ case 'o':
+ if (!OptionEnable(optarg, options, config))
return 1;
break;
case 'C':
options->working_dir = optarg;
break;
+ case 'p':
+ options->persistent = true;
+ break;
case OPT_VERSION:
printf("%s\n", kNinjaVersion);
return 0;
+ case OPT_FRONTEND:
+ config->frontend = optarg;
+ break;
+ case OPT_FRONTEND_FILE:
+ config->frontend_file = optarg;
+ break;
case 'h':
default:
Usage(*config);
@@ -1201,9 +1493,30 @@
*argv += optind;
*argc -= optind;
+ if (config->frontend != NULL && config->frontend_file != NULL) {
+ Fatal("only one of --frontend or --frontend_file may be specified.");
+ }
+
+ if (config->pre_remove_output_files && !config->uses_phony_outputs) {
+ Fatal("preremoveoutputs=yes requires usesphonyoutputs=yes.");
+ }
+
return -1;
}
+static void WaitForInput(const BuildConfig& config) {
+ static char* buf = nullptr;
+ static size_t len = 0;
+
+ if (config.verbosity == BuildConfig::VERBOSE) {
+ fprintf(stderr, "ninja waiting for input...\n");
+ }
+ ssize_t rc = getline(&buf, &len, stdin);
+ if (rc < 0) {
+ exit(0);
+ }
+}
+
NORETURN void real_main(int argc, char** argv) {
// Use exit() instead of return in this function to avoid potentially
// expensive cleanup when destructing NinjaMain.
@@ -1231,12 +1544,15 @@
// Don't print this if a tool is being used, so that tool output
// can be piped into a file without this string showing up.
if (!options.tool)
- printf("ninja: Entering directory `%s'\n", options.working_dir);
+ Info("Entering directory `%s'", options.working_dir);
if (chdir(options.working_dir) < 0) {
- Fatal("chdir to '%s' - %s", options.working_dir, strerror(errno));
+ Error("chdir to '%s' - %s", options.working_dir, strerror(errno));
+ exit(1);
}
}
+ SetThreadPoolThreadCount(g_use_threads ? GetProcessorCount() : 1);
+
if (options.tool && options.tool->when == Tool::RUN_AFTER_FLAGS) {
// None of the RUN_AFTER_FLAGS actually use a NinjaMain, but it's needed
// by other tools.
@@ -1244,11 +1560,22 @@
exit((ninja.*options.tool->func)(&options, argc, argv));
}
+ Status* status = NULL;
+
// Limit number of rebuilds, to prevent infinite loops.
const int kCycleLimit = 100;
for (int cycle = 1; cycle <= kCycleLimit; ++cycle) {
NinjaMain ninja(ninja_command, config);
+ if (status == NULL) {
+#ifndef _WIN32
+ if (config.frontend != NULL || config.frontend_file != NULL)
+ status = new StatusSerializer(config);
+ else
+#endif
+ status = new StatusPrinter(config);
+ }
+
ManifestParserOptions parser_opts;
if (options.dupe_edges_should_err) {
parser_opts.dupe_edge_action_ = kDupeEdgeActionError;
@@ -1259,7 +1586,7 @@
ManifestParser parser(&ninja.state_, &ninja.disk_interface_, parser_opts);
string err;
if (!parser.Load(options.input_file, &err)) {
- Error("%s", err.c_str());
+ status->Error("%s", err.c_str());
exit(1);
}
@@ -1276,7 +1603,7 @@
exit((ninja.*options.tool->func)(&options, argc, argv));
// Attempt to rebuild the manifest before building anything else
- if (ninja.RebuildManifest(options.input_file, &err)) {
+ if (ninja.RebuildManifest(options.input_file, &err, status)) {
// In dry_run mode the regeneration will succeed without changing the
// manifest forever. Better to return immediately.
if (config.dry_run)
@@ -1284,18 +1611,37 @@
// Start the build over with the new manifest.
continue;
} else if (!err.empty()) {
- Error("rebuilding '%s': %s", options.input_file, err.c_str());
+ status->Error("rebuilding '%s': %s", options.input_file, err.c_str());
exit(1);
}
- int result = ninja.RunBuild(argc, argv);
+ int result = 0;
+ do {
+ Stopwatch stopwatch;
+
+ if (options.persistent) {
+ WaitForInput(config);
+ stopwatch.Restart();
+ }
+
+ result = ninja.RunBuild(argc, argv, status);
+ if (options.persistent) {
+ fprintf(stderr, "build %s in %0.3f seconds\n",
+ result == 0 ? "succeeded" : "failed", stopwatch.Elapsed());
+ ninja.state_.Reset();
+ }
+ } while (options.persistent);
+
if (g_metrics)
- ninja.DumpMetrics();
+ ninja.DumpMetrics(status);
+
+ delete status;
exit(result);
}
- Error("manifest '%s' still dirty after %d tries\n",
+ status->Error("manifest '%s' still dirty after %d tries",
options.input_file, kCycleLimit);
+ delete status;
exit(1);
}
diff --git a/src/ninja_test.cc b/src/ninja_test.cc
index d642c5c..066ed43 100644
--- a/src/ninja_test.cc
+++ b/src/ninja_test.cc
@@ -25,8 +25,9 @@
#include <getopt.h>
#endif
-#include "test.h"
#include "line_printer.h"
+#include "test.h"
+#include "thread_pool.h"
struct RegisteredTest {
testing::Test* (*factory)();
@@ -104,6 +105,9 @@
*test_filter = optarg;
break;
} // else fall through.
+#if defined(__has_cpp_attribute) && __has_cpp_attribute(clang::fallthrough)
+ [[clang::fallthrough]];
+#endif
default:
Usage();
return false;
@@ -129,6 +133,8 @@
int main(int argc, char **argv) {
int tests_started = 0;
+ SetThreadPoolThreadCount(16);
+
const char* test_filter = "*";
if (!ReadFlags(&argc, &argv, &test_filter))
return 1;
diff --git a/src/parallel_map.h b/src/parallel_map.h
new file mode 100644
index 0000000..00799d8
--- /dev/null
+++ b/src/parallel_map.h
@@ -0,0 +1,109 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_PARALLEL_MAP_H_
+#define NINJA_PARALLEL_MAP_H_
+
+#include <functional>
+#include <mutex>
+#include <utility>
+#include <vector>
+
+#include "thread_pool.h"
+#include "util.h"
+
+namespace detail {
+
+template <typename T>
+struct MapFnResults {
+ MapFnResults(size_t size) : size_(size), results_(new T[size] {}) {}
+
+ using ReturnType = std::vector<T>;
+
+ template <typename Vector, typename MapFn>
+ void MapItem(Vector& vec, const MapFn& map_fn, size_t idx) {
+ assert(idx < size_);
+ results_.get()[idx] = map_fn(vec[idx]);
+ }
+
+ ReturnType Finish() {
+ ReturnType result;
+ result.reserve(size_);
+ std::move(results_.get(), results_.get() + size_,
+ std::back_inserter(result));
+ return result;
+ }
+
+private:
+ size_t size_;
+
+ /// We'd like to store the intermediate results using std::vector<T>, but
+ /// the std::vector<bool> specialization doesn't allow concurrent modification
+ /// of different elements, so use a simple array instead.
+ std::unique_ptr<T[]> results_;
+};
+
+template <>
+struct MapFnResults<void> {
+ MapFnResults(size_t) {}
+
+ using ReturnType = void;
+
+ template <typename Vector, typename MapFn>
+ void MapItem(Vector& vec, const MapFn& map_fn, size_t idx) {
+ map_fn(vec[idx]);
+ }
+
+ ReturnType Finish() {}
+};
+
+} // namespace detail
+
+template <typename T>
+std::vector<std::pair<T, T>> SplitByThreads(T total) {
+ return SplitByCount(total, GetOptimalThreadPoolJobCount());
+}
+
+/// Run |map_fn| on each element of |vec| and return a vector containing the
+/// result, in the same order as the input vector. Returns void instead if the
+/// map function returns void.
+///
+/// The map function is invoked from multiple threads, so the functor is marked
+/// const. (e.g. A "mutable" lambda isn't allowed.)
+template <typename Vector, typename MapFn>
+auto ParallelMap(ThreadPool* thread_pool, Vector&& vec, const MapFn& map_fn) ->
+ typename detail::MapFnResults<
+ typename std::remove_reference<
+ decltype(map_fn(vec[0]))>::type>::ReturnType {
+ // Identify the return type of the map function, and if it isn't void, prepare
+ // a vector to hold the result of mapping each item in the sequence.
+ using MapReturn = typename std::remove_reference<decltype(map_fn(vec[0]))>::type;
+ detail::MapFnResults<MapReturn> results(vec.size());
+
+ // Split the sequence up into groups.
+ std::vector<std::pair<size_t, size_t>> ranges = SplitByThreads(vec.size());
+ std::vector<std::function<void()>> tasks;
+ for (auto& range : ranges) {
+ tasks.emplace_back([&results, &vec, &map_fn, range] {
+ for (size_t idx = range.first; idx < range.second; ++idx) {
+ results.MapItem(vec, map_fn, idx);
+ }
+ });
+ }
+ thread_pool->RunTasks(std::move(tasks));
+
+ return results.Finish();
+}
+
+#endif // NINJA_PARALLEL_MAP_H_
diff --git a/src/parser.cc b/src/parser.cc
index 745c532..b483a6e 100644
--- a/src/parser.cc
+++ b/src/parser.cc
@@ -29,23 +29,16 @@
return false;
}
- // The lexer needs a nul byte at the end of its input, to know when it's done.
- // It takes a StringPiece, and StringPiece's string constructor uses
- // string::data(). data()'s return value isn't guaranteed to be
- // null-terminated (although in practice - libc++, libstdc++, msvc's stl --
- // it is, and C++11 demands that too), so add an explicit nul byte.
- contents.resize(contents.size() + 1);
-
return Parse(filename, contents, err);
}
-bool Parser::ExpectToken(Lexer::Token expected, string* err) {
- Lexer::Token token = lexer_.ReadToken();
+bool Parser::ExpectToken(Lexer& lexer, Lexer::Token expected, string* err) {
+ Lexer::Token token = lexer.ReadToken();
if (token != expected) {
string message = string("expected ") + Lexer::TokenName(expected);
message += string(", got ") + Lexer::TokenName(token);
message += Lexer::TokenErrorHint(expected);
- return lexer_.Error(message, err);
+ return lexer.Error(message, err);
}
return true;
}
diff --git a/src/parser.h b/src/parser.h
index e2d2b97..78b2d00 100644
--- a/src/parser.h
+++ b/src/parser.h
@@ -35,11 +35,10 @@
protected:
/// If the next token is not \a expected, produce an error string
/// saying "expected foo, got bar".
- bool ExpectToken(Lexer::Token expected, string* err);
+ bool ExpectToken(Lexer& lexer, Lexer::Token expected, string* err);
State* state_;
FileReader* file_reader_;
- Lexer lexer_;
private:
/// Parse a file, given its contents as a string.
diff --git a/src/proto.cc b/src/proto.cc
new file mode 100644
index 0000000..c29e621
--- /dev/null
+++ b/src/proto.cc
@@ -0,0 +1,154 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#define __STDC_LIMIT_MACROS
+
+#include <inttypes.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "proto.h"
+
+// This file and proto.h implement a minimal write-only protobuf runtime to be
+// used with headers generated by misc/generate_proto_header.py.
+
+// From https://github.com/google/protobuf/blob/3.0.x/src/google/protobuf/wire_format_lite.h
+enum WireType {
+ WIRETYPE_VARINT = 0,
+ WIRETYPE_FIXED64 = 1,
+ WIRETYPE_LENGTH_DELIMITED = 2,
+ WIRETYPE_START_GROUP = 3,
+ WIRETYPE_END_GROUP = 4,
+ WIRETYPE_FIXED32 = 5,
+};
+
+static uint8_t MakeTag(int number, WireType wire_type) {
+ const size_t FIELD_NUMBER_SHIFT = 3;
+ return (number << FIELD_NUMBER_SHIFT) | static_cast<uint8_t>(wire_type);
+}
+
+// Based on https://github.com/google/protobuf/blob/3.0.x/src/google/protobuf/io/coded_stream.h
+// Compile-time equivalent of VarintSize64().
+template <uint64_t Value>
+struct StaticVarintSize {
+ static const int value =
+ (Value < (1ULL << 7))
+ ? 1
+ : (Value < (1ULL << 14))
+ ? 2
+ : (Value < (1ULL << 21))
+ ? 3
+ : (Value < (1ULL << 28))
+ ? 4
+ : (Value < (1ULL << 35))
+ ? 5
+ : (Value < (1ULL << 42))
+ ? 6
+ : (Value < (1ULL << 49))
+ ? 7
+ : (Value < (1ULL << 56))
+ ? 8
+ : (Value < (1ULL << 63))
+ ? 9
+ : 10;
+};
+
+static void WriteVarint32WithType(std::ostream* output, int number,
+ uint32_t value, WireType type) {
+ uint8_t buf[StaticVarintSize<UINT32_MAX>::value + 1];
+ uint8_t *target = buf;
+
+ *target++ = MakeTag(number, type);
+
+ while (value >= 0x80) {
+ *target = static_cast<uint8_t>(value | 0x80);
+ value >>= 7;
+ ++target;
+ }
+ *target++ = static_cast<uint8_t>(value);
+
+ output->write(reinterpret_cast<char*>(buf), target - buf);
+}
+
+void WriteVarint32NoTag(std::ostream* output, uint32_t value) {
+ uint8_t buf[StaticVarintSize<UINT32_MAX>::value];
+ uint8_t *target = buf;
+
+ while (value >= 0x80) {
+ *target = static_cast<uint8_t>(value | 0x80);
+ value >>= 7;
+ ++target;
+ }
+ *target++ = static_cast<uint8_t>(value);
+
+ output->write(reinterpret_cast<char*>(buf), target - buf);
+}
+
+void WriteVarint32(std::ostream* output, int number, uint32_t value) {
+ WriteVarint32WithType(output, number, value, WIRETYPE_VARINT);
+}
+
+void WriteVarint32SignExtended(std::ostream* output, int number,
+ int32_t value) {
+ if (value < 0) {
+ WriteVarint64(output, number, static_cast<uint64_t>(value));
+ } else {
+ WriteVarint32(output, number, static_cast<uint32_t>(value));
+ }
+}
+
+void WriteVarint64(std::ostream* output, int number, uint64_t value) {
+ uint8_t buf[StaticVarintSize<UINT64_MAX>::value + 1];
+ uint8_t *target = buf;
+
+ *target++ = MakeTag(number, WIRETYPE_VARINT);
+
+ while (value >= 0x80) {
+ *target = static_cast<uint8_t>(value | 0x80);
+ value >>= 7;
+ ++target;
+ }
+ *target++ = static_cast<uint8_t>(value);
+
+ output->write(reinterpret_cast<char*>(buf), target - buf);
+}
+
+void WriteFixed32(std::ostream* output, int number, uint32_t value) {
+ uint8_t buf[sizeof(value) + 1];
+ uint8_t *target = buf;
+
+ *target++ = MakeTag(number, WIRETYPE_FIXED32);
+ memcpy(target, &value, sizeof(value));
+
+ output->write(reinterpret_cast<char*>(buf), sizeof(buf));
+}
+
+void WriteFixed64(std::ostream* output, int number, uint64_t value) {
+ uint8_t buf[sizeof(value) + 1];
+ uint8_t *target = buf;
+
+ *target++ = MakeTag(number, WIRETYPE_FIXED64);
+ memcpy(target, &value, sizeof(value));
+
+ output->write(reinterpret_cast<char*>(buf), sizeof(buf));
+}
+
+void WriteString(std::ostream* output, int number, const std::string& value) {
+ WriteLengthDelimited(output, number, value.size());
+ output->write(value.data(), value.size());
+}
+
+void WriteLengthDelimited(std::ostream* output, int number, size_t size) {
+ WriteVarint32WithType(output, number, size, WIRETYPE_LENGTH_DELIMITED);
+}
diff --git a/src/proto.h b/src/proto.h
new file mode 100644
index 0000000..8552719
--- /dev/null
+++ b/src/proto.h
@@ -0,0 +1,110 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_PROTO_H_
+#define NINJA_PROTO_H_
+
+#include <iostream>
+
+// This file and proto.cc implement a minimal write-only protobuf runtime to be
+// used with headers generated by misc/generate_proto_header.py.
+
+// Based on https://github.com/google/protobuf/blob/3.0.x/src/google/protobuf/wire_format_lite.h
+static inline uint32_t ZigZagEncode32(int32_t n) {
+ // Note: the right-shift must be arithmetic
+ return (static_cast<uint32_t>(n) << 1) ^ (n >> 31);
+}
+
+static inline uint64_t ZigZagEncode64(int64_t n) {
+ // Note: the right-shift must be arithmetic
+ return (static_cast<uint64_t>(n) << 1) ^ (n >> 63);
+}
+
+// Based on https://github.com/google/protobuf/blob/3.0.x/src/google/protobuf/io/coded_stream.h
+static inline size_t VarintSize32(uint32_t value) {
+ if (value < (1 << 7)) {
+ return 1;
+ } else if (value < (1 << 14)) {
+ return 2;
+ } else if (value < (1 << 21)) {
+ return 3;
+ } else if (value < (1 << 28)) {
+ return 4;
+ } else {
+ return 5;
+ }
+}
+
+static inline size_t VarintSize32SignExtended(int32_t value) {
+ if (value < 0) {
+ return 10; // TODO(kenton): Make this a symbolic constant.
+ } else {
+ return VarintSize32(static_cast<uint32_t>(value));
+ }
+}
+
+static inline size_t VarintSize64(uint64_t value) {
+ if (value < (1ull << 35)) {
+ if (value < (1ull << 7)) {
+ return 1;
+ } else if (value < (1ull << 14)) {
+ return 2;
+ } else if (value < (1ull << 21)) {
+ return 3;
+ } else if (value < (1ull << 28)) {
+ return 4;
+ } else {
+ return 5;
+ }
+ } else {
+ if (value < (1ull << 42)) {
+ return 6;
+ } else if (value < (1ull << 49)) {
+ return 7;
+ } else if (value < (1ull << 56)) {
+ return 8;
+ } else if (value < (1ull << 63)) {
+ return 9;
+ } else {
+ return 10;
+ }
+ }
+}
+
+static inline size_t VarintSizeBool(bool /*value*/) {
+ return 1;
+}
+
+static inline size_t FixedSize32(uint32_t /*value*/) {
+ return 4;
+}
+
+static inline size_t FixedSize64(uint64_t /*value*/) {
+ return 8;
+}
+
+static inline size_t StringSize(const std::string& value) {
+ return VarintSize32(value.size()) + value.size();
+}
+
+void WriteVarint32(std::ostream* output, int number, uint32_t value);
+void WriteVarint32NoTag(std::ostream* output, uint32_t value);
+void WriteVarint32SignExtended(std::ostream* output, int number, int32_t value);
+void WriteVarint64(std::ostream* output, int number, uint64_t value);
+void WriteFixed32(std::ostream* output, int number, uint32_t value);
+void WriteFixed64(std::ostream* output, int number, uint64_t value);
+void WriteString(std::ostream* output, int number, const std::string& value);
+void WriteLengthDelimited(std::ostream* output, int number, size_t size);
+
+#endif
diff --git a/src/state.cc b/src/state.cc
index 74cf4c1..98059dc 100644
--- a/src/state.cc
+++ b/src/state.cc
@@ -22,6 +22,9 @@
#include "metrics.h"
#include "util.h"
+Pool::Pool(const HashedStrView& name, int depth) : name_(name), depth_(depth) {
+ pos_.base = new BasePosition {{ &State::kBuiltinScope, 0 }}; // leaked
+}
void Pool::EdgeScheduled(const Edge& edge) {
if (depth_ != 0)
@@ -38,7 +41,7 @@
delayed_.insert(edge);
}
-void Pool::RetrieveReadyEdges(set<Edge*>* ready_queue) {
+void Pool::RetrieveReadyEdges(EdgeSet* ready_queue) {
DelayedEdges::iterator it = delayed_.begin();
while (it != delayed_.end()) {
Edge* edge = *it;
@@ -61,63 +64,77 @@
}
}
-// static
-bool Pool::WeightedEdgeCmp(const Edge* a, const Edge* b) {
- if (!a) return b;
- if (!b) return false;
- int weight_diff = a->weight() - b->weight();
- return ((weight_diff < 0) || (weight_diff == 0 && a < b));
-}
-
+Scope State::kBuiltinScope({});
Pool State::kDefaultPool("", 0);
Pool State::kConsolePool("console", 1);
-const Rule State::kPhonyRule("phony");
+Rule State::kPhonyRule("phony");
State::State() {
- bindings_.AddRule(&kPhonyRule);
+ // Reserve scope position (root, 0) for built-in rules.
+ root_scope_.AllocDecls(1);
+
+ AddBuiltinRule(&kPhonyRule);
AddPool(&kDefaultPool);
AddPool(&kConsolePool);
}
-void State::AddPool(Pool* pool) {
- assert(LookupPool(pool->name()) == NULL);
- pools_[pool->name()] = pool;
+// Add a built-in rule at the top of the root scope.
+void State::AddBuiltinRule(Rule* rule) {
+ root_scope_.AddRule(rule);
}
-Pool* State::LookupPool(const string& pool_name) {
- map<string, Pool*>::iterator i = pools_.find(pool_name);
- if (i == pools_.end())
- return NULL;
- return i->second;
+bool State::AddPool(Pool* pool) {
+ return pools_.insert({ pool->name_hashed(), pool }).second;
}
Edge* State::AddEdge(const Rule* rule) {
Edge* edge = new Edge();
+ edge->pos_.base = new BasePosition {{ &root_scope_, 0 }}; // leaked
edge->rule_ = rule;
edge->pool_ = &State::kDefaultPool;
- edge->env_ = &bindings_;
+ edge->id_ = edges_.size();
edges_.push_back(edge);
return edge;
}
-Node* State::GetNode(StringPiece path, uint64_t slash_bits) {
- Node* node = LookupNode(path);
- if (node)
- return node;
- node = new Node(path.AsString(), slash_bits);
- paths_[node->path()] = node;
- return node;
+Pool* State::LookupPool(const HashedStrView& pool_name) {
+ auto i = pools_.find(pool_name);
+ if (i == pools_.end())
+ return nullptr;
+ return i->second;
}
-Node* State::LookupNode(StringPiece path) const {
- METRIC_RECORD("lookup node");
- Paths::const_iterator i = paths_.find(path);
- if (i != paths_.end())
- return i->second;
- return NULL;
+Pool* State::LookupPoolAtPos(const HashedStrView& pool_name,
+ DeclIndex dfs_location) {
+ Pool* result = LookupPool(pool_name);
+ if (result == nullptr) return nullptr;
+ return result->dfs_location() < dfs_location ? result : nullptr;
}
-Node* State::SpellcheckNode(const string& path) {
+Node* State::GetNode(const HashedStrView& path, uint64_t slash_bits) {
+ if (Node** opt_node = paths_.Lookup(path))
+ return *opt_node;
+ // Create a new node and try to insert it.
+ std::unique_ptr<Node> node(new Node(path, slash_bits));
+ if (paths_.insert({node->path_hashed(), node.get()}).second)
+ return node.release();
+ // Another thread beat us to it. Use its node instead.
+ return *paths_.Lookup(path);
+}
+
+Node* State::LookupNode(const HashedStrView& path) const {
+ if (Node* const* opt_node = paths_.Lookup(path))
+ return *opt_node;
+ return nullptr;
+}
+
+Node* State::LookupNodeAtPos(const HashedStrView& path,
+ DeclIndex dfs_location) const {
+ Node* result = LookupNode(path);
+ return result && result->dfs_location() < dfs_location ? result : nullptr;
+}
+
+Node* State::SpellcheckNode(StringPiece path) {
const bool kAllowReplacements = true;
const int kMaxValidEditDistance = 3;
@@ -125,7 +142,7 @@
Node* result = NULL;
for (Paths::iterator i = paths_.begin(); i != paths_.end(); ++i) {
int distance = EditDistance(
- i->first, path, kAllowReplacements, kMaxValidEditDistance);
+ i->first.str_view(), path, kAllowReplacements, kMaxValidEditDistance);
if (distance < min_distance && i->second) {
min_distance = distance;
result = i->second;
@@ -149,16 +166,6 @@
return true;
}
-bool State::AddDefault(StringPiece path, string* err) {
- Node* node = LookupNode(path);
- if (!node) {
- *err = "unknown target '" + path.AsString() + "'";
- return false;
- }
- defaults_.push_back(node);
- return true;
-}
-
vector<Node*> State::RootNodes(string* err) const {
vector<Node*> root_nodes;
// Search for nodes with no output.
@@ -166,7 +173,7 @@
e != edges_.end(); ++e) {
for (vector<Node*>::const_iterator out = (*e)->outputs_.begin();
out != (*e)->outputs_.end(); ++out) {
- if ((*out)->out_edges().empty())
+ if (!(*out)->has_out_edge())
root_nodes.push_back(*out);
}
}
@@ -181,6 +188,12 @@
return defaults_.empty() ? RootNodes(err) : defaults_;
}
+DeclIndex State::AllocDfsLocation(DeclIndex count) {
+ DeclIndex result = dfs_location_;
+ dfs_location_ += count;
+ return result;
+}
+
void State::Reset() {
for (Paths::iterator i = paths_.begin(); i != paths_.end(); ++i)
i->second->ResetState();
@@ -188,6 +201,7 @@
(*e)->outputs_ready_ = false;
(*e)->deps_loaded_ = false;
(*e)->mark_ = Edge::VisitNone;
+ (*e)->precomputed_dirtiness_ = false;
}
}
@@ -202,9 +216,7 @@
}
if (!pools_.empty()) {
printf("resource_pools:\n");
- for (map<string, Pool*>::const_iterator it = pools_.begin();
- it != pools_.end(); ++it)
- {
+ for (auto it = pools_.begin(); it != pools_.end(); ++it) {
if (!it->second->name().empty()) {
it->second->Dump();
}
diff --git a/src/state.h b/src/state.h
index 6fe886c..db18d60 100644
--- a/src/state.h
+++ b/src/state.h
@@ -16,13 +16,16 @@
#define NINJA_STATE_H_
#include <map>
+#include <memory>
#include <set>
#include <string>
#include <vector>
using namespace std;
#include "eval_env.h"
-#include "hash_map.h"
+#include "graph.h"
+#include "string_piece.h"
+#include "concurrent_hash_map.h"
#include "util.h"
struct Edge;
@@ -38,13 +41,17 @@
/// the total scheduled weight diminishes enough (i.e. when a scheduled edge
/// completes).
struct Pool {
- Pool(const string& name, int depth)
- : name_(name), current_use_(0), depth_(depth), delayed_(&WeightedEdgeCmp) {}
+ Pool() {}
+
+ /// Constructor for built-in pools.
+ Pool(const HashedStrView& name, int depth);
// A depth of 0 is infinite
bool is_valid() const { return depth_ >= 0; }
int depth() const { return depth_; }
- const string& name() const { return name_; }
+ DeclIndex dfs_location() const { return pos_.dfs_location(); }
+ const std::string& name() const { return name_.str(); }
+ const HashedStr& name_hashed() const { return name_; }
int current_use() const { return current_use_; }
/// true if the Pool might delay this edge
@@ -62,45 +69,78 @@
void DelayEdge(Edge* edge);
/// Pool will add zero or more edges to the ready_queue
- void RetrieveReadyEdges(set<Edge*>* ready_queue);
+ void RetrieveReadyEdges(EdgeSet* ready_queue);
/// Dump the Pool and its edges (useful for debugging).
void Dump() const;
- private:
- string name_;
+ HashedStr name_;
+ int depth_ = 0;
+ RelativePosition pos_;
+ // Temporary fields used only during manifest parsing.
+ struct {
+ size_t pool_name_diag_pos = 0;
+ StringPiece depth;
+ size_t depth_diag_pos = 0;
+ } parse_state_;
+
+private:
/// |current_use_| is the total of the weights of the edges which are
/// currently scheduled in the Plan (i.e. the edges in Plan::ready_).
- int current_use_;
- int depth_;
+ int current_use_ = 0;
- static bool WeightedEdgeCmp(const Edge* a, const Edge* b);
+ struct WeightedEdgeCmp {
+ bool operator()(const Edge* a, const Edge* b) const {
+ if (!a) return b;
+ if (!b) return false;
+ int weight_diff = a->weight() - b->weight();
+ return ((weight_diff < 0) || (weight_diff == 0 && EdgeCmp()(a, b)));
+ }
+ };
- typedef set<Edge*,bool(*)(const Edge*, const Edge*)> DelayedEdges;
+ typedef set<Edge*, WeightedEdgeCmp> DelayedEdges;
DelayedEdges delayed_;
};
/// Global state (file status) for a single run.
struct State {
+ /// The built-in pools and rules use this dummy built-in scope to initialize
+ /// their RelativePosition fields. The scope won't have anything in it.
+ static Scope kBuiltinScope;
+
static Pool kDefaultPool;
static Pool kConsolePool;
- static const Rule kPhonyRule;
+ static Rule kPhonyRule;
State();
- void AddPool(Pool* pool);
- Pool* LookupPool(const string& pool_name);
-
+ void AddBuiltinRule(Rule* rule);
+ bool AddPool(Pool* pool);
Edge* AddEdge(const Rule* rule);
+ Pool* LookupPool(const HashedStrView& pool_name);
- Node* GetNode(StringPiece path, uint64_t slash_bits);
- Node* LookupNode(StringPiece path) const;
- Node* SpellcheckNode(const string& path);
+ /// Lookup a pool at a DFS position. Pools don't respect scopes. A pool's name
+ /// must be unique across the entire manifest, and it must be declared before
+ /// an edge references it.
+ Pool* LookupPoolAtPos(const HashedStrView& pool_name, DeclIndex dfs_location);
+ /// Creates the node if it doesn't exist. Never returns nullptr. Thread-safe.
+ /// The hash table should be resized ahead of time for decent performance.
+ Node* GetNode(const HashedStrView& path, uint64_t slash_bits);
+
+ /// Finds the existing node, returns nullptr if it doesn't exist yet.
+ /// Thread-safe.
+ Node* LookupNode(const HashedStrView& path) const;
+ Node* LookupNodeAtPos(const HashedStrView& path,
+ DeclIndex dfs_location) const;
+
+ Node* SpellcheckNode(StringPiece path);
+
+ /// These methods aren't thread-safe.
void AddIn(Edge* edge, StringPiece path, uint64_t slash_bits);
bool AddOut(Edge* edge, StringPiece path, uint64_t slash_bits);
- bool AddDefault(StringPiece path, string* error);
+ void AddDefault(Node* node) { defaults_.push_back(node); }
/// Reset state. Keeps all nodes and edges, but restores them to the
/// state where we haven't yet examined the disk for dirty state.
@@ -114,18 +154,27 @@
vector<Node*> RootNodes(string* error) const;
vector<Node*> DefaultNodes(string* error) const;
+ DeclIndex AllocDfsLocation(DeclIndex count);
+
/// Mapping of path -> Node.
- typedef ExternalStringHashMap<Node*>::Type Paths;
+ ///
+ /// This hash map uses a value type of Node* rather than Node, which allows the
+ /// map entry's key (a string view) to refer to the Node::name field.
+ typedef ConcurrentHashMap<HashedStrView, Node*> Paths;
Paths paths_;
/// All the pools used in the graph.
- map<string, Pool*> pools_;
+ std::unordered_map<HashedStrView, Pool*> pools_;
/// All the edges of the graph.
vector<Edge*> edges_;
- BindingEnv bindings_;
+ Scope root_scope_ { ScopePosition {} };
vector<Node*> defaults_;
+
+private:
+ /// Position 0 is used for built-in decls (e.g. pools).
+ DeclIndex dfs_location_ = 1;
};
#endif // NINJA_STATE_H_
diff --git a/src/state_test.cc b/src/state_test.cc
index 458b519..d9e800d 100644
--- a/src/state_test.cc
+++ b/src/state_test.cc
@@ -21,15 +21,9 @@
TEST(State, Basic) {
State state;
- EvalString command;
- command.AddText("cat ");
- command.AddSpecial("in");
- command.AddText(" > ");
- command.AddSpecial("out");
-
Rule* rule = new Rule("cat");
- rule->AddBinding("command", command);
- state.bindings_.AddRule(rule);
+ rule->bindings_.emplace_back("command", "cat $in > $out\n");
+ state.root_scope_.AddRule(rule);
Edge* edge = state.AddEdge(rule);
state.AddIn(edge, "in1", 0);
diff --git a/src/status.cc b/src/status.cc
new file mode 100644
index 0000000..c5a37e8
--- /dev/null
+++ b/src/status.cc
@@ -0,0 +1,439 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "status.h"
+
+#include <errno.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "debug_flags.h"
+
+StatusPrinter::StatusPrinter(const BuildConfig& config)
+ : config_(config),
+ started_edges_(0), finished_edges_(0), total_edges_(0), running_edges_(0),
+ time_millis_(0), progress_status_format_(NULL),
+ current_rate_(config.parallelism) {
+
+ // Don't do anything fancy in verbose mode.
+ if (config_.verbosity != BuildConfig::NORMAL)
+ printer_.set_smart_terminal(false);
+
+ progress_status_format_ = getenv("NINJA_STATUS");
+ if (!progress_status_format_)
+ progress_status_format_ = "[%f/%t] ";
+}
+
+void StatusPrinter::PlanHasTotalEdges(int total) {
+ total_edges_ = total;
+}
+
+void StatusPrinter::BuildEdgeStarted(Edge* edge, int64_t start_time_millis) {
+ ++started_edges_;
+ ++running_edges_;
+ time_millis_ = start_time_millis;
+
+ if (edge->use_console() || printer_.is_smart_terminal())
+ PrintStatus(edge, start_time_millis);
+
+ if (edge->use_console())
+ printer_.SetConsoleLocked(true);
+}
+
+void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t end_time_millis,
+ const CommandRunner::Result* result) {
+ time_millis_ = end_time_millis;
+ ++finished_edges_;
+
+ if (edge->use_console())
+ printer_.SetConsoleLocked(false);
+
+ if (config_.verbosity == BuildConfig::QUIET)
+ return;
+
+ if (!edge->use_console())
+ PrintStatus(edge, end_time_millis);
+
+ --running_edges_;
+
+ // Print the command that is spewing before printing its output.
+ if (!result->success()) {
+ string outputs;
+ for (vector<Node*>::const_iterator o = edge->outputs_.begin();
+ o != edge->outputs_.end(); ++o)
+ outputs += (*o)->path() + " ";
+
+ printer_.PrintOnNewLine("FAILED: " + outputs + "\n");
+ printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n");
+ }
+
+ if (!result->output.empty()) {
+ // ninja sets stdout and stderr of subprocesses to a pipe, to be able to
+ // check if the output is empty. Some compilers, e.g. clang, check
+ // isatty(stderr) to decide if they should print colored output.
+ // To make it possible to use colored output with ninja, subprocesses should
+ // be run with a flag that forces them to always print color escape codes.
+ // To make sure these escape codes don't show up in a file if ninja's output
+ // is piped to a file, ninja strips ansi escape codes again if it's not
+ // writing to a |smart_terminal_|.
+ // (Launching subprocesses in pseudo ttys doesn't work because there are
+ // only a few hundred available on some systems, and ninja can launch
+ // thousands of parallel compile commands.)
+ string final_output;
+ if (!printer_.supports_color())
+ final_output = StripAnsiEscapeCodes(result->output);
+ else
+ final_output = result->output;
+
+#ifdef _WIN32
+ // Fix extra CR being added on Windows, writing out CR CR LF (#773)
+ _setmode(_fileno(stdout), _O_BINARY); // Begin Windows extra CR fix
+#endif
+
+ printer_.PrintOnNewLine(final_output);
+
+#ifdef _WIN32
+ _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix
+#endif
+ }
+}
+
+void StatusPrinter::BuildLoadDyndeps() {
+ // The DependencyScan calls EXPLAIN() to print lines explaining why
+ // it considers a portion of the graph to be out of date. Normally
+ // this is done before the build starts, but our caller is about to
+ // load a dyndep file during the build. Doing so may generate more
+ // exlanation lines (via fprintf directly to stderr), but in an
+ // interactive console the cursor is currently at the end of a status
+ // line. Start a new line so that the first explanation does not
+ // append to the status line. After the explanations are done a
+ // new build status line will appear.
+ if (g_explaining)
+ printer_.PrintOnNewLine("");
+}
+
+void StatusPrinter::BuildStarted() {
+ started_edges_ = 0;
+ finished_edges_ = 0;
+ running_edges_ = 0;
+}
+
+void StatusPrinter::BuildFinished() {
+ printer_.SetConsoleLocked(false);
+ printer_.PrintOnNewLine("");
+}
+
+string StatusPrinter::FormatProgressStatus(const char* progress_status_format,
+ int64_t time_millis) const {
+ string out;
+ char buf[32];
+ for (const char* s = progress_status_format; *s != '\0'; ++s) {
+ if (*s == '%') {
+ ++s;
+ switch (*s) {
+ case '%':
+ out.push_back('%');
+ break;
+
+ // Started edges.
+ case 's':
+ snprintf(buf, sizeof(buf), "%d", started_edges_);
+ out += buf;
+ break;
+
+ // Total edges.
+ case 't':
+ snprintf(buf, sizeof(buf), "%d", total_edges_);
+ out += buf;
+ break;
+
+ // Running edges.
+ case 'r': {
+ snprintf(buf, sizeof(buf), "%d", running_edges_);
+ out += buf;
+ break;
+ }
+
+ // Unstarted edges.
+ case 'u':
+ snprintf(buf, sizeof(buf), "%d", total_edges_ - started_edges_);
+ out += buf;
+ break;
+
+ // Finished edges.
+ case 'f':
+ snprintf(buf, sizeof(buf), "%d", finished_edges_);
+ out += buf;
+ break;
+
+ // Overall finished edges per second.
+ case 'o':
+ SnprintfRate(finished_edges_ / (time_millis_ / 1e3), buf, "%.1f");
+ out += buf;
+ break;
+
+ // Current rate, average over the last '-j' jobs.
+ case 'c':
+ current_rate_.UpdateRate(finished_edges_, time_millis_);
+ SnprintfRate(current_rate_.rate(), buf, "%.1f");
+ out += buf;
+ break;
+
+ // Percentage
+ case 'p': {
+ int percent = (100 * finished_edges_) / total_edges_;
+ snprintf(buf, sizeof(buf), "%3i%%", percent);
+ out += buf;
+ break;
+ }
+
+ case 'e': {
+ snprintf(buf, sizeof(buf), "%.3f", time_millis_ / 1e3);
+ out += buf;
+ break;
+ }
+
+ default:
+ Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s);
+ return "";
+ }
+ } else {
+ out.push_back(*s);
+ }
+ }
+
+ return out;
+}
+
+void StatusPrinter::PrintStatus(Edge* edge, int64_t time_millis) {
+ if (config_.verbosity == BuildConfig::QUIET)
+ return;
+
+ bool force_full_command = config_.verbosity == BuildConfig::VERBOSE;
+
+ string to_print = edge->GetBinding("description");
+ if (to_print.empty() || force_full_command)
+ to_print = edge->GetBinding("command");
+
+ to_print = FormatProgressStatus(progress_status_format_, time_millis)
+ + to_print;
+
+ printer_.Print(to_print,
+ force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE);
+}
+
+void StatusPrinter::Warning(const char* msg, ...) {
+ va_list ap;
+ va_start(ap, msg);
+ ::Warning(msg, ap);
+ va_end(ap);
+}
+
+void StatusPrinter::Error(const char* msg, ...) {
+ va_list ap;
+ va_start(ap, msg);
+ ::Error(msg, ap);
+ va_end(ap);
+}
+
+void StatusPrinter::Info(const char* msg, ...) {
+ va_list ap;
+ va_start(ap, msg);
+ ::Info(msg, ap);
+ va_end(ap);
+}
+
+void StatusPrinter::Debug(const char* msg, ...) {
+ va_list ap;
+ va_start(ap, msg);
+ ::Info(msg, ap);
+ va_end(ap);
+}
+
+#ifndef _WIN32
+
+#include "frontend.pb.h"
+#include "proto.h"
+
+StatusSerializer::StatusSerializer(const BuildConfig& config) :
+ config_(config), subprocess_(NULL), total_edges_(0) {
+ if (config.frontend != NULL) {
+ int output_pipe[2];
+ if (pipe(output_pipe) < 0)
+ Fatal("pipe: %s", strerror(errno));
+ SetCloseOnExec(output_pipe[1]);
+
+ f_ = fdopen(output_pipe[1], "wb");
+
+ // Launch the frontend as a subprocess with write-end of the pipe as fd 3
+ subprocess_ = subprocess_set_.Add(config.frontend, /*use_console=*/true,
+ output_pipe[0]);
+ close(output_pipe[0]);
+ } else if (config.frontend_file != NULL) {
+ f_ = fopen(config.frontend_file, "wb");
+ if (!f_) {
+ Fatal("fopen: %s", strerror(errno));
+ }
+ SetCloseOnExec(fileno(f_));
+ } else {
+ Fatal("No output specified for serialization");
+ }
+
+ setvbuf(f_, NULL, _IONBF, 0);
+ filebuf_ = new ofilebuf(f_);
+ out_ = new std::ostream(filebuf_);
+}
+
+StatusSerializer::~StatusSerializer() {
+ delete out_;
+ delete filebuf_;
+ fclose(f_);
+ if (subprocess_ != NULL) {
+ subprocess_->Finish();
+ subprocess_set_.Clear();
+ }
+}
+
+void StatusSerializer::Send() {
+ // Send the proto as a length-delimited message
+ WriteVarint32NoTag(out_, proto_.ByteSizeLong());
+ proto_.SerializeToOstream(out_);
+ proto_.Clear();
+ out_->flush();
+}
+
+void StatusSerializer::PlanHasTotalEdges(int total) {
+ if (total_edges_ != total) {
+ total_edges_ = total;
+ ninja::Status::TotalEdges *total_edges = proto_.mutable_total_edges();
+ total_edges->set_total_edges(total_edges_);
+ Send();
+ }
+}
+
+void StatusSerializer::BuildEdgeStarted(Edge* edge, int64_t start_time_millis) {
+ ninja::Status::EdgeStarted* edge_started = proto_.mutable_edge_started();
+
+ edge_started->set_id(edge->id_);
+ edge_started->set_start_time(start_time_millis);
+
+ edge_started->mutable_inputs()->reserve(edge->inputs_.size());
+ for (vector<Node*>::iterator it = edge->inputs_.begin();
+ it != edge->inputs_.end(); ++it) {
+ edge_started->add_inputs((*it)->path());
+ }
+
+ edge_started->mutable_outputs()->reserve(edge->inputs_.size());
+ for (vector<Node*>::iterator it = edge->outputs_.begin();
+ it != edge->outputs_.end(); ++it) {
+ edge_started->add_outputs((*it)->path());
+ }
+
+ edge_started->set_desc(edge->GetBinding("description"));
+
+ edge_started->set_command(edge->GetBinding("command"));
+
+ edge_started->set_console(edge->use_console());
+
+ Send();
+}
+
+uint32_t timeval_to_ms(struct timeval tv) {
+ return tv.tv_sec * 1000 + tv.tv_usec / 1000;
+}
+
+void StatusSerializer::BuildEdgeFinished(Edge* edge, int64_t end_time_millis,
+ const CommandRunner::Result* result) {
+ ninja::Status::EdgeFinished* edge_finished = proto_.mutable_edge_finished();
+
+ edge_finished->set_id(edge->id_);
+ edge_finished->set_end_time(end_time_millis);
+ edge_finished->set_status(result->status);
+ edge_finished->set_output(result->output);
+ edge_finished->set_user_time(timeval_to_ms(result->rusage.ru_utime));
+ edge_finished->set_system_time(timeval_to_ms(result->rusage.ru_stime));
+
+ Send();
+}
+
+void StatusSerializer::BuildLoadDyndeps() {}
+
+void StatusSerializer::BuildStarted() {
+ ninja::Status::BuildStarted* build_started = proto_.mutable_build_started();
+
+ build_started->set_parallelism(config_.parallelism);
+ build_started->set_verbose((config_.verbosity == BuildConfig::VERBOSE));
+
+ Send();
+}
+
+void StatusSerializer::BuildFinished() {
+ proto_.mutable_build_finished();
+ Send();
+}
+
+void StatusSerializer::Message(ninja::Status::Message::Level level,
+ const char* msg, va_list ap) {
+ va_list ap2;
+ va_copy(ap2, ap);
+
+ int len = vsnprintf(NULL, 0, msg, ap2);
+ if (len < 0) {
+ Fatal("vsnprintf failed");
+ }
+
+ va_end(ap2);
+
+ string buf;
+ buf.resize(len + 1);
+
+ len = vsnprintf(&buf[0], len + 1, msg, ap);
+ buf.resize(len);
+
+ ninja::Status::Message* message = proto_.mutable_message();
+
+ message->set_level(level);
+ message->set_message(buf);
+
+ Send();
+}
+
+void StatusSerializer::Debug(const char* msg, ...) {
+ va_list ap;
+ va_start(ap, msg);
+ Message(ninja::Status::Message::DEBUG, msg, ap);
+ va_end(ap);
+}
+
+void StatusSerializer::Info(const char* msg, ...) {
+ va_list ap;
+ va_start(ap, msg);
+ Message(ninja::Status::Message::INFO, msg, ap);
+ va_end(ap);
+}
+
+void StatusSerializer::Warning(const char* msg, ...) {
+ va_list ap;
+ va_start(ap, msg);
+ Message(ninja::Status::Message::WARNING, msg, ap);
+ va_end(ap);
+}
+
+void StatusSerializer::Error(const char* msg, ...) {
+ va_list ap;
+ va_start(ap, msg);
+ Message(ninja::Status::Message::ERROR, msg, ap);
+ va_end(ap);
+}
+#endif // !_WIN32
diff --git a/src/status.h b/src/status.h
new file mode 100644
index 0000000..ba6381b
--- /dev/null
+++ b/src/status.h
@@ -0,0 +1,166 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_STATUS_H_
+#define NINJA_STATUS_H_
+
+#include <stdarg.h>
+
+#include <map>
+#include <string>
+using namespace std;
+
+#include "build.h"
+#include "line_printer.h"
+#include "subprocess.h"
+
+/// Abstract interface to object that tracks the status of a build:
+/// completion fraction, printing updates.
+struct Status {
+ virtual void PlanHasTotalEdges(int total) = 0;
+ virtual void BuildEdgeStarted(Edge* edge, int64_t start_time_millis) = 0;
+ virtual void BuildEdgeFinished(Edge* edge, int64_t end_time_millis,
+ const CommandRunner::Result* result) = 0;
+ virtual void BuildLoadDyndeps() = 0;
+ virtual void BuildStarted() = 0;
+ virtual void BuildFinished() = 0;
+
+ virtual void Debug(const char* msg, ...) = 0;
+ virtual void Info(const char* msg, ...) = 0;
+ virtual void Warning(const char* msg, ...) = 0;
+ virtual void Error(const char* msg, ...) = 0;
+
+ virtual ~Status() { }
+};
+
+/// Implementation of the Status interface that prints the status as
+/// human-readable strings to stdout
+struct StatusPrinter : Status {
+ explicit StatusPrinter(const BuildConfig& config);
+ virtual void PlanHasTotalEdges(int total);
+ virtual void BuildEdgeStarted(Edge* edge, int64_t start_time_millis);
+ virtual void BuildEdgeFinished(Edge* edge, int64_t end_time_millis,
+ const CommandRunner::Result* result);
+ virtual void BuildLoadDyndeps();
+ virtual void BuildStarted();
+ virtual void BuildFinished();
+
+ virtual void Debug(const char* msg, ...);
+ virtual void Info(const char* msg, ...);
+ virtual void Warning(const char* msg, ...);
+ virtual void Error(const char* msg, ...);
+
+ virtual ~StatusPrinter() { }
+
+ /// Format the progress status string by replacing the placeholders.
+ /// See the user manual for more information about the available
+ /// placeholders.
+ /// @param progress_status_format The format of the progress status.
+ /// @param status The status of the edge.
+ string FormatProgressStatus(const char* progress_status_format,
+ int64_t time_millis) const;
+
+ private:
+ void PrintStatus(Edge* edge, int64_t time_millis);
+
+ const BuildConfig& config_;
+
+ int started_edges_, finished_edges_, total_edges_, running_edges_;
+ int64_t time_millis_;
+
+ /// Prints progress output.
+ LinePrinter printer_;
+
+ /// The custom progress status format to use.
+ const char* progress_status_format_;
+
+ template<size_t S>
+ void SnprintfRate(double rate, char(&buf)[S], const char* format) const {
+ if (rate == -1)
+ snprintf(buf, S, "?");
+ else
+ snprintf(buf, S, format, rate);
+ }
+
+ struct SlidingRateInfo {
+ SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {}
+
+ double rate() { return rate_; }
+
+ void UpdateRate(int update_hint, int64_t time_millis_) {
+ if (update_hint == last_update_)
+ return;
+ last_update_ = update_hint;
+
+ if (times_.size() == N)
+ times_.pop();
+ times_.push(time_millis_);
+ if (times_.back() != times_.front())
+ rate_ = times_.size() / ((times_.back() - times_.front()) / 1e3);
+ }
+
+ private:
+ double rate_;
+ const size_t N;
+ queue<double> times_;
+ int last_update_;
+ };
+
+ mutable SlidingRateInfo current_rate_;
+};
+
+#ifndef _WIN32
+
+#include "filebuf.h"
+#include "frontend.pb.h"
+
+/// Implementation of the Status interface that serializes the status as
+/// protocol buffer messages to a subprocess
+struct StatusSerializer : Status {
+ explicit StatusSerializer(const BuildConfig& config);
+ virtual ~StatusSerializer();
+
+ virtual void PlanHasTotalEdges(int total);
+ virtual void BuildEdgeStarted(Edge* edge, int64_t start_time);
+ virtual void BuildEdgeFinished(Edge* edge, int64_t end_time_millis,
+ const CommandRunner::Result* result);
+ virtual void BuildLoadDyndeps();
+ virtual void BuildStarted();
+ virtual void BuildFinished();
+
+ virtual void Debug(const char* msg, ...);
+ virtual void Info(const char* msg, ...);
+ virtual void Warning(const char* msg, ...);
+ virtual void Error(const char* msg, ...);
+
+ const BuildConfig& config_;
+
+ FILE* f_;
+ ofilebuf *filebuf_;
+ std::ostream *out_;
+
+ ninja::Status proto_;
+
+ SubprocessSet subprocess_set_;
+ Subprocess* subprocess_;
+
+ int total_edges_;
+private:
+ void Message(ninja::Status::Message::Level level, const char* msg, va_list ap);
+ void Send();
+};
+
+#endif // !_WIN32
+
+#endif // NINJA_STATUS_H_
diff --git a/src/status_test.cc b/src/status_test.cc
new file mode 100644
index 0000000..6e42490
--- /dev/null
+++ b/src/status_test.cc
@@ -0,0 +1,35 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "status.h"
+
+#include "test.h"
+
+TEST(StatusTest, StatusFormatElapsed) {
+ BuildConfig config;
+ StatusPrinter status(config);
+
+ status.BuildStarted();
+ // Before any task is done, the elapsed time must be zero.
+ EXPECT_EQ("[%/e0.000]",
+ status.FormatProgressStatus("[%%/e%e]", 0));
+}
+
+TEST(StatusTest, StatusFormatReplacePlaceholder) {
+ BuildConfig config;
+ StatusPrinter status(config);
+
+ EXPECT_EQ("[%/s0/t0/r0/u0/f0]",
+ status.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]", 0));
+}
diff --git a/src/string_piece.h b/src/string_piece.h
index 031bda4..c233b0a 100644
--- a/src/string_piece.h
+++ b/src/string_piece.h
@@ -15,32 +15,31 @@
#ifndef NINJA_STRINGPIECE_H_
#define NINJA_STRINGPIECE_H_
+#include <string.h>
+
+#include <algorithm>
#include <string>
-using namespace std;
+#include "hash_map.h"
-#include <string.h>
+using namespace std;
/// StringPiece represents a slice of a string whose memory is managed
/// externally. It is useful for reducing the number of std::strings
/// we need to allocate.
+///
+/// This class is something like std::string_view from C++17.
struct StringPiece {
typedef const char* const_iterator;
+ typedef const char& const_reference;
- StringPiece() : str_(NULL), len_(0) {}
+ constexpr StringPiece() : str_(nullptr), len_(0) {}
/// The constructors intentionally allow for implicit conversions.
StringPiece(const string& str) : str_(str.data()), len_(str.size()) {}
StringPiece(const char* str) : str_(str), len_(strlen(str)) {}
- StringPiece(const char* str, size_t len) : str_(str), len_(len) {}
-
- bool operator==(const StringPiece& other) const {
- return len_ == other.len_ && memcmp(str_, other.str_, len_) == 0;
- }
- bool operator!=(const StringPiece& other) const {
- return !(*this == other);
- }
+ constexpr StringPiece(const char* str, size_t len) : str_(str), len_(len) {}
/// Convert the slice into a full-fledged std::string, copying the
/// data into a new string.
@@ -48,24 +47,67 @@
return len_ ? string(str_, len_) : string();
}
- const_iterator begin() const {
- return str_;
+ constexpr const_iterator begin() const { return str_; }
+ constexpr const_iterator end() const { return str_ + len_; }
+ constexpr const char* data() const { return str_; }
+ constexpr const_reference operator[](size_t pos) const { return str_[pos]; }
+ constexpr size_t size() const { return len_; }
+ constexpr bool empty() const { return len_ == 0; }
+ constexpr const_reference front() const { return str_[0]; }
+ constexpr const_reference back() const { return str_[len_ - 1]; }
+
+ void remove_prefix(size_t n) { str_ += n; len_ -= n; }
+ void remove_suffix(size_t n) { len_ -= n; }
+
+ StringPiece substr(size_t pos=0, size_t count=std::string::npos) const {
+ return StringPiece(str_ + pos, std::min(count, len_ - pos));
}
- const_iterator end() const {
- return str_ + len_;
- }
-
- char operator[](size_t pos) const {
- return str_[pos];
- }
-
- size_t size() const {
- return len_;
+ int compare(StringPiece other) const {
+ size_t min_len = std::min(len_, other.len_);
+ if (min_len != 0) { // strncmp(NULL, NULL, 0) has undefined behavior.
+ if (int cmp = strncmp(str_, other.str_, min_len)) {
+ return cmp;
+ }
+ }
+ return (len_ == other.len_) ? 0 : ((len_ < other.len_) ? -1 : 1);
}
const char* str_;
size_t len_;
};
+inline bool operator==(StringPiece x, StringPiece y) {
+ if (x.size() != y.size())
+ return false;
+ if (x.size() == 0) // memcmp(NULL, NULL, 0) has undefined behavior.
+ return true;
+ return !memcmp(x.data(), y.data(), x.size());
+}
+
+inline bool operator!=(StringPiece x, StringPiece y) {
+ return !(x == y);
+}
+
+inline bool operator<(StringPiece x, StringPiece y) { return x.compare(y) < 0; }
+inline bool operator>(StringPiece x, StringPiece y) { return x.compare(y) > 0; }
+inline bool operator<=(StringPiece x, StringPiece y) { return x.compare(y) <= 0; }
+inline bool operator>=(StringPiece x, StringPiece y) { return x.compare(y) >= 0; }
+
+inline uint32_t HashStr(const StringPiece& str) {
+ if (str.empty()) {
+ // Returning 0 for an empty string allows HashedStrView to be zero-initialized.
+ return 0;
+ }
+ return MurmurHash2(str.data(), str.size());
+}
+
+namespace std {
+ template<> struct hash<StringPiece> {
+ size_t operator()(StringPiece key) const {
+ return HashStr(key);
+ }
+ };
+}
+
#endif // NINJA_STRINGPIECE_H_
diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc
index fc5543e..d7c1a13 100644
--- a/src/subprocess-posix.cc
+++ b/src/subprocess-posix.cc
@@ -41,7 +41,8 @@
Finish();
}
-bool Subprocess::Start(SubprocessSet* set, const string& command) {
+bool Subprocess::Start(SubprocessSet* set, const string& command,
+ int extra_fd) {
int output_pipe[2];
if (pipe(output_pipe) < 0)
Fatal("pipe: %s", strerror(errno));
@@ -63,6 +64,11 @@
if (err != 0)
Fatal("posix_spawn_file_actions_addclose: %s", strerror(err));
+ if (extra_fd >= 0) {
+ if (posix_spawn_file_actions_adddup2(&action, extra_fd, 3) != 0)
+ Fatal("posix_spawn_file_actions_adddup2: %s", strerror(errno));
+ }
+
posix_spawnattr_t attr;
err = posix_spawnattr_init(&attr);
if (err != 0)
@@ -140,11 +146,15 @@
}
}
+const struct rusage* Subprocess::GetUsage() const {
+ return &rusage_;
+}
+
ExitStatus Subprocess::Finish() {
assert(pid_ != -1);
int status;
- if (waitpid(pid_, &status, 0) < 0)
- Fatal("waitpid(%d): %s", pid_, strerror(errno));
+ if (wait4(pid_, &status, 0, &rusage_) < 0)
+ Fatal("wait4(%d): %s", pid_, strerror(errno));
pid_ = -1;
if (WIFEXITED(status)) {
@@ -221,9 +231,10 @@
Fatal("sigprocmask: %s", strerror(errno));
}
-Subprocess *SubprocessSet::Add(const string& command, bool use_console) {
+Subprocess *SubprocessSet::Add(const string& command, bool use_console,
+ int extra_fd) {
Subprocess *subprocess = new Subprocess(use_console);
- if (!subprocess->Start(this, command)) {
+ if (!subprocess->Start(this, command, extra_fd)) {
delete subprocess;
return 0;
}
diff --git a/src/subprocess.h b/src/subprocess.h
index b2d486c..db8a030 100644
--- a/src/subprocess.h
+++ b/src/subprocess.h
@@ -24,6 +24,7 @@
#include <windows.h>
#else
#include <signal.h>
+#include <sys/resource.h>
#endif
// ppoll() exists on FreeBSD, but only on newer versions.
@@ -51,9 +52,13 @@
const string& GetOutput() const;
+#ifndef _WIN32
+ const struct rusage* GetUsage() const;
+#endif
+
private:
Subprocess(bool use_console);
- bool Start(struct SubprocessSet* set, const string& command);
+ bool Start(struct SubprocessSet* set, const string& command, int extra_fd);
void OnPipeReady();
string buf_;
@@ -71,6 +76,7 @@
#else
int fd_;
pid_t pid_;
+ struct rusage rusage_;
#endif
bool use_console_;
@@ -84,7 +90,8 @@
SubprocessSet();
~SubprocessSet();
- Subprocess* Add(const string& command, bool use_console = false);
+ Subprocess* Add(const string& command, bool use_console = false,
+ int extra_fd = -1);
bool DoWork();
Subprocess* NextFinished();
void Clear();
diff --git a/src/test.cc b/src/test.cc
index a9816bc..86983cc 100644
--- a/src/test.cc
+++ b/src/test.cc
@@ -109,35 +109,63 @@
}
void VerifyGraph(const State& state) {
- for (vector<Edge*>::const_iterator e = state.edges_.begin();
- e != state.edges_.end(); ++e) {
+ std::set<Node*> edge_nodes;
+ std::map<Node*, Edge*> expected_node_input;
+ std::map<Node*, std::set<Edge*>> expected_node_outputs;
+
+ // An edge ID is always equal to its index in the global edge table.
+ for (size_t idx = 0; idx < state.edges_.size(); ++idx)
+ EXPECT_EQ(idx, state.edges_[idx]->id_);
+
+ // Check that each edge is valid.
+ for (Edge* e : state.edges_) {
// All edges need at least one output.
- EXPECT_FALSE((*e)->outputs_.empty());
- // Check that the edge's inputs have the edge as out-edge.
- for (vector<Node*>::const_iterator in_node = (*e)->inputs_.begin();
- in_node != (*e)->inputs_.end(); ++in_node) {
- const vector<Edge*>& out_edges = (*in_node)->out_edges();
- EXPECT_NE(find(out_edges.begin(), out_edges.end(), *e),
- out_edges.end());
+ EXPECT_FALSE(e->outputs_.empty());
+ // Check that the edge's node counts are correct.
+ EXPECT_GE(e->explicit_outs_, 0);
+ EXPECT_GE(e->implicit_outs_, 0);
+ EXPECT_GE(e->explicit_deps_, 0);
+ EXPECT_GE(e->implicit_deps_, 0);
+ EXPECT_GE(e->order_only_deps_, 0);
+ EXPECT_EQ(static_cast<size_t>(e->explicit_outs_ + e->implicit_outs_),
+ e->outputs_.size());
+ EXPECT_EQ(static_cast<size_t>(e->explicit_deps_ + e->implicit_deps_ +
+ e->order_only_deps_),
+ e->inputs_.size());
+ // Record everything seen in edges so we can check it against the nodes.
+ for (Node* n : e->inputs_) {
+ edge_nodes.insert(n);
+ expected_node_outputs[n].insert(e);
}
- // Check that the edge's outputs have the edge as in-edge.
- for (vector<Node*>::const_iterator out_node = (*e)->outputs_.begin();
- out_node != (*e)->outputs_.end(); ++out_node) {
- EXPECT_EQ((*out_node)->in_edge(), *e);
+ for (Node* n : e->outputs_) {
+ edge_nodes.insert(n);
+ EXPECT_EQ(nullptr, expected_node_input[n]);
+ expected_node_input[n] = e;
}
}
- // The union of all in- and out-edges of each nodes should be exactly edges_.
- set<const Edge*> node_edge_set;
- for (State::Paths::const_iterator p = state.paths_.begin();
- p != state.paths_.end(); ++p) {
- const Node* n = p->second;
- if (n->in_edge())
- node_edge_set.insert(n->in_edge());
- node_edge_set.insert(n->out_edges().begin(), n->out_edges().end());
+ // Verify that any node used by an edge is in the node table.
+ //
+ // N.B. It's OK to have nodes that aren't referenced by any edge. The code
+ // below will verify that such nodes have no in/out edges. This situation can
+ // happen in a couple of ways:
+ // - Loading the depslog creates a node for every path in the log, but the
+ // implicit edge<->node links are only created as ninja scans targets for
+ // dirty nodes.
+ // - Removing an invalid edge for dupbuild=warn doesn't remove the input
+ // nodes.
+ for (Node* n : edge_nodes)
+ EXPECT_EQ(n, state.LookupNode(n->path()));
+
+ // Check each node against the set of edges.
+ for (const auto& p : state.paths_) {
+ Node* n = p.second;
+ auto out_edges_vec = n->GetOutEdges();
+ std::set<Edge*> out_edges(out_edges_vec.begin(), out_edges_vec.end());
+ EXPECT_EQ(n, state.LookupNode(n->path()));
+ EXPECT_EQ(expected_node_outputs[n], out_edges);
+ EXPECT_EQ(expected_node_input[n], n->in_edge());
}
- set<const Edge*> edge_set(state.edges_.begin(), state.edges_.end());
- EXPECT_EQ(node_edge_set, edge_set);
}
void VirtualFileSystem::Create(const string& path,
@@ -147,21 +175,90 @@
files_created_.insert(path);
}
+void VirtualFileSystem::CreateSymlink(const string& path,
+ const string& dest) {
+ files_[path].mtime = now_;
+ files_[path].contents = dest;
+ files_[path].is_symlink = true;
+ files_created_.insert(path);
+}
+
TimeStamp VirtualFileSystem::Stat(const string& path, string* err) const {
+ DirMap::const_iterator d = dirs_.find(path);
+ if (d != dirs_.end()) {
+ *err = d->second.stat_error;
+ return d->second.mtime;
+ }
FileMap::const_iterator i = files_.find(path);
if (i != files_.end()) {
+ if (i->second.is_symlink) {
+ return Stat(i->second.contents, err);
+ }
*err = i->second.stat_error;
return i->second.mtime;
}
return 0;
}
+TimeStamp VirtualFileSystem::LStat(const string& path, bool* is_dir, string* err) const {
+ DirMap::const_iterator d = dirs_.find(path);
+ if (d != dirs_.end()) {
+ if (is_dir != nullptr)
+ *is_dir = true;
+ *err = d->second.stat_error;
+ return d->second.mtime;
+ }
+ FileMap::const_iterator i = files_.find(path);
+ if (i != files_.end()) {
+ if (is_dir != nullptr)
+ *is_dir = false;
+ *err = i->second.stat_error;
+ return i->second.mtime;
+ }
+ return 0;
+}
+
+bool VirtualFileSystem::IsStatThreadSafe() const {
+ return true;
+}
+
bool VirtualFileSystem::WriteFile(const string& path, const string& contents) {
+ if (files_.find(path) == files_.end()) {
+ if (dirs_.find(path) != dirs_.end())
+ return false;
+
+ string::size_type slash_pos = path.find_last_of("/");
+ if (slash_pos != string::npos) {
+ DirMap::iterator d = dirs_.find(path.substr(0, slash_pos));
+ if (d != dirs_.end()) {
+ d->second.mtime = now_;
+ } else {
+ return false;
+ }
+ }
+ }
+
Create(path, contents);
return true;
}
bool VirtualFileSystem::MakeDir(const string& path) {
+ if (dirs_.find(path) != dirs_.end())
+ return true;
+ if (files_.find(path) != files_.end())
+ return false;
+
+ string::size_type slash_pos = path.find_last_of("/");
+ if (slash_pos != string::npos) {
+ DirMap::iterator d = dirs_.find(path.substr(0, slash_pos));
+ if (d != dirs_.end()) {
+ d->second.mtime = now_;
+ } else {
+ return false;
+ }
+ }
+
+ dirs_[path].mtime = now_;
directories_made_.push_back(path);
return true; // success
}
@@ -169,10 +266,27 @@
FileReader::Status VirtualFileSystem::ReadFile(const string& path,
string* contents,
string* err) {
+ // Delegate to the more-general LoadFile.
+ std::unique_ptr<LoadedFile> file;
+ Status result = LoadFile(path, &file, err);
+ if (result == Okay) {
+ *contents = file->content().AsString();
+ }
+ return result;
+}
+
+FileReader::Status VirtualFileSystem::LoadFile(const std::string& path,
+ std::unique_ptr<LoadedFile>* result,
+ std::string* err) {
files_read_.push_back(path);
FileMap::iterator i = files_.find(path);
if (i != files_.end()) {
- *contents = i->second.contents;
+ if (i->second.is_symlink) {
+ return LoadFile(i->second.contents, result, err);
+ }
+ std::string& contents = i->second.contents;
+ *result = std::unique_ptr<HeapLoadedFile>(new HeapLoadedFile(path,
+ contents));
return Okay;
}
*err = strerror(ENOENT);
@@ -180,11 +294,18 @@
}
int VirtualFileSystem::RemoveFile(const string& path) {
- if (find(directories_made_.begin(), directories_made_.end(), path)
- != directories_made_.end())
+ if (dirs_.find(path) != dirs_.end())
return -1;
FileMap::iterator i = files_.find(path);
if (i != files_.end()) {
+ string::size_type slash_pos = path.find_last_of("/");
+ if (slash_pos != string::npos) {
+ DirMap::iterator d = dirs_.find(path.substr(0, slash_pos));
+ if (d != dirs_.end()) {
+ d->second.mtime = now_;
+ }
+ }
+
files_.erase(i);
files_removed_.insert(path);
return 0;
diff --git a/src/test.h b/src/test.h
index 6af17b3..5d49b0a 100644
--- a/src/test.h
+++ b/src/test.h
@@ -137,6 +137,9 @@
/// "Create" a file with contents.
void Create(const string& path, const string& contents);
+ /// "Create" a symlink pointing to another path.
+ void CreateSymlink(const string& path, const string& dest);
+
/// Tick "time" forwards; subsequent file operations will be newer than
/// previous ones.
int Tick() {
@@ -145,9 +148,14 @@
// DiskInterface
virtual TimeStamp Stat(const string& path, string* err) const;
+ virtual TimeStamp LStat(const string& path, bool* is_dir, string* err) const;
+ virtual bool IsStatThreadSafe() const;
virtual bool WriteFile(const string& path, const string& contents);
virtual bool MakeDir(const string& path);
virtual Status ReadFile(const string& path, string* contents, string* err);
+ virtual Status LoadFile(const std::string& path,
+ std::unique_ptr<LoadedFile>* result,
+ std::string* err);
virtual int RemoveFile(const string& path);
/// An entry for a single in-memory file.
@@ -155,10 +163,17 @@
int mtime;
string stat_error; // If mtime is -1.
string contents;
+ bool is_symlink = false;
+ };
+ struct DirEntry {
+ int mtime;
+ string stat_error; // If mtime is -1.
};
vector<string> directories_made_;
vector<string> files_read_;
+ typedef map<string, DirEntry> DirMap;
+ DirMap dirs_;
typedef map<string, Entry> FileMap;
FileMap files_;
set<string> files_removed_;
@@ -181,4 +196,9 @@
string temp_dir_name_;
};
+// A C++11 version of operator "" s from C++14's std::string_literals.
+inline std::string operator "" _s(const char* str, size_t len) {
+ return { str, len };
+}
+
#endif // NINJA_TEST_H_
diff --git a/src/thread_pool.cc b/src/thread_pool.cc
new file mode 100644
index 0000000..30ec7ad
--- /dev/null
+++ b/src/thread_pool.cc
@@ -0,0 +1,200 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "thread_pool.h"
+
+#include <assert.h>
+
+#include <atomic>
+#include <condition_variable>
+#include <thread>
+
+#include "metrics.h"
+#include "util.h"
+
+struct ThreadPoolImpl : ThreadPool {
+ ThreadPoolImpl(int num_threads);
+ virtual ~ThreadPoolImpl();
+ void RunTasks(std::vector<std::function<void()>>&& tasks) override;
+
+private:
+ void WorkerThreadLoop();
+
+ struct Batch {
+ Batch(size_t task_count, std::vector<std::function<void()>>&& tasks)
+ : task_count(task_count), tasks(std::move(tasks)) {}
+
+ const size_t task_count;
+ std::atomic<size_t> next_task_idx { 0 };
+ std::atomic<size_t> tasks_completed { 0 };
+ std::vector<std::function<void()>> tasks;
+ bool completed = false;
+ };
+
+ std::mutex mutex_;
+ std::shared_ptr<Batch> current_batch_;
+ bool shutting_down_ = false;
+
+ std::condition_variable worker_cv_;
+ std::condition_variable main_cv_;
+
+ std::vector<std::thread> threads_;
+};
+
+ThreadPoolImpl::ThreadPoolImpl(int num_threads) {
+ if (num_threads <= 1) {
+ // On a single-core machine (or with "-d nothreads"), don't start any
+ // threads.
+ return;
+ }
+
+ threads_.resize(num_threads);
+ for (int i = 0; i < num_threads; ++i) {
+ threads_[i] = std::thread([this] {
+ WorkerThreadLoop();
+ });
+ }
+}
+
+ThreadPoolImpl::~ThreadPoolImpl() {
+ {
+ std::lock_guard<std::mutex> lock(mutex_);
+ shutting_down_ = true;
+ }
+ worker_cv_.notify_all();
+ for (auto& thread : threads_) {
+ thread.join();
+ }
+}
+
+void ThreadPoolImpl::RunTasks(std::vector<std::function<void()>>&& tasks) {
+ // Sometimes it's better to run the tasks on the caller's thread:
+ // - When parsing the manifest tree, only one file is parsed at a time, so if
+ // a manifest tree has many small files, the overhead of dispatching each
+ // file to a worker thread can be substantial.
+ // - When ninja runs on a single-core machine, or with "-d nothreads", there
+ // are no worker threads in the pool.
+ const size_t task_count = tasks.size();
+ if (threads_.empty() || task_count <= 1) {
+ for (auto& task : tasks) {
+ task();
+ }
+ return;
+ }
+
+ std::shared_ptr<Batch> batch = std::make_shared<Batch>(task_count,
+ std::move(tasks));
+
+ {
+ std::lock_guard<std::mutex> lock(mutex_);
+ assert(current_batch_.get() == nullptr &&
+ "the thread pool isn't intended for reentrant or concurrent use");
+ current_batch_ = batch;
+ }
+
+ worker_cv_.notify_all();
+
+ {
+ std::unique_lock<std::mutex> lock(mutex_);
+ main_cv_.wait(lock, [batch]() { return batch->completed; });
+ current_batch_ = {};
+ }
+
+ // The client's std::function might do interesting work when it's destructed
+ // (e.g. destruct a lambda's captured state), so destruct the functions before
+ // returning. It's safe to modify the batch's tasks vector because there are
+ // no more tasks to run. The Batch itself is freed at an unpredictable time.
+ batch->tasks.clear();
+}
+
+void ThreadPoolImpl::WorkerThreadLoop() {
+ while (true) {
+ // A shared_ptr object isn't thread-safe itself, but its control block is
+ // thread-safe. This worker thread must lock the mutex before checking the
+ // active batch.
+ std::shared_ptr<Batch> batch;
+
+ {
+ // Wait until either:
+ // - There is an active batch with at least one task remaining.
+ // - The thread pool is shutting down.
+ std::unique_lock<std::mutex> lock(mutex_);
+ worker_cv_.wait(lock, [this, &batch]() {
+ auto b = current_batch_;
+ if (b && b->next_task_idx.load() < b->task_count) {
+ batch = b;
+ return true;
+ }
+ return shutting_down_;
+ });
+ if (shutting_down_) {
+ return;
+ }
+ }
+
+ while (true) {
+ // Try to start another task in this batch. Atomically load and increment
+ // the next task index.
+ size_t idx = batch->next_task_idx++;
+ if (idx >= batch->task_count) {
+ // The fetched next-task-index is expected to exceed the total number of
+ // tasks as the threads finish the batch. Return to the main loop. Some
+ // other thread will finish the batch, if it hasn't already.
+ break;
+ }
+
+ batch->tasks[idx]();
+
+ // Atomically increment the number of completed tasks. Exactly one worker
+ // thread should notice that the completed task count equals the total
+ // number of tasks, and that worker thread signals the main thread.
+ if (++batch->tasks_completed >= batch->task_count) {
+ assert(batch->tasks_completed.load() == batch->task_count &&
+ "BatchThreadPool::Batch completion count exceeded task count");
+
+ // Every task is completed, so mark the batch done and wake up the main
+ // thread.
+ {
+ std::lock_guard<std::mutex> lock(mutex_);
+
+ batch->completed = true;
+ }
+ main_cv_.notify_one();
+ break;
+ }
+ }
+ }
+}
+
+static int g_num_threads = 1;
+
+void SetThreadPoolThreadCount(int num_threads) {
+ g_num_threads = std::max(num_threads, 1);
+}
+
+int GetOptimalThreadPoolJobCount() {
+ if (g_num_threads > 1) {
+ // Magic constant: when splitting work into tasks for the thread pool, try
+ // to create a fixed number of tasks per thread in the pool.
+ return g_num_threads * 2;
+ } else {
+ // If there are no worker threads, then multiple tasks aren't useful.
+ // Returning 1 will disable manifest and log file splitting.
+ return 1;
+ }
+}
+
+std::unique_ptr<ThreadPool> CreateThreadPool() {
+ return std::unique_ptr<ThreadPool>(new ThreadPoolImpl(g_num_threads));
+}
diff --git a/src/thread_pool.h b/src/thread_pool.h
new file mode 100644
index 0000000..9091647
--- /dev/null
+++ b/src/thread_pool.h
@@ -0,0 +1,45 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_THREAD_POOL_H_
+#define NINJA_THREAD_POOL_H_
+
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include "util.h"
+
+/// This class maintains a pool of threads and distributes tasks across them.
+/// It only executes one batch at a time. It is mostly lock-free, and threads
+/// retrieve tasks using atomic size_t fields.
+struct ThreadPool {
+ virtual ~ThreadPool() {}
+ virtual void RunTasks(std::vector<std::function<void()>>&& tasks) = 0;
+};
+
+/// Configure the number of worker threads a thread pool creates. A value of 1
+/// or lower disables thread creation.
+void SetThreadPoolThreadCount(int num_threads);
+
+/// Get the optimal number of jobs to split work into, given the size of the
+/// thread pool.
+int GetOptimalThreadPoolJobCount();
+
+/// Create a new thread pool. Destructing the thread pool joins the threads,
+/// which is important for ensuring that no worker threads are running when
+/// ninja forks child processes or handles signals.
+std::unique_ptr<ThreadPool> CreateThreadPool();
+
+#endif // NINJA_THREAD_POOL_H_
diff --git a/src/util.cc b/src/util.cc
index ee810d6..bb91923 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -35,9 +35,11 @@
#ifndef _WIN32
#include <unistd.h>
+#include <sys/mman.h>
#include <sys/time.h>
#endif
+#include <atomic>
#include <vector>
#if defined(__APPLE__) || defined(__FreeBSD__)
@@ -53,10 +55,10 @@
#include "edit_distance.h"
#include "metrics.h"
+#include "string_piece.h"
void Fatal(const char* msg, ...) {
va_list ap;
- fprintf(stderr, "ninja: fatal: ");
va_start(ap, msg);
vfprintf(stderr, msg, ap);
va_end(ap);
@@ -72,24 +74,45 @@
#endif
}
+void Warning(const char* msg, va_list ap) {
+ fprintf(stderr, "ninja: warning: ");
+ vfprintf(stderr, msg, ap);
+ fprintf(stderr, "\n");
+}
+
void Warning(const char* msg, ...) {
va_list ap;
- fprintf(stderr, "ninja: warning: ");
va_start(ap, msg);
- vfprintf(stderr, msg, ap);
+ Warning(msg, ap);
va_end(ap);
+}
+
+void Error(const char* msg, va_list ap) {
+ fprintf(stderr, "ninja: error: ");
+ vfprintf(stderr, msg, ap);
fprintf(stderr, "\n");
}
void Error(const char* msg, ...) {
va_list ap;
- fprintf(stderr, "ninja: error: ");
va_start(ap, msg);
- vfprintf(stderr, msg, ap);
+ Error(msg, ap);
va_end(ap);
+}
+
+void Info(const char* msg, va_list ap) {
+ fprintf(stderr, "ninja: ");
+ vfprintf(stderr, msg, ap);
fprintf(stderr, "\n");
}
+void Info(const char* msg, ...) {
+ va_list ap;
+ va_start(ap, msg);
+ Info(msg, ap);
+ va_end(ap);
+}
+
bool CanonicalizePath(string* path, uint64_t* slash_bits, string* err) {
METRIC_RECORD("canonicalize str");
size_t len = path->size();
@@ -372,6 +395,59 @@
#endif
}
+#ifndef _WIN32
+void Mapping::unmap() {
+ if (base_ != nullptr) {
+ if (munmap(base_, mapping_size_) < 0)
+ perror("munmap");
+ base_ = nullptr;
+ file_size_ = 0;
+ mapping_size_ = 0;
+ }
+}
+
+int MapFile(const std::string& filename,
+ std::unique_ptr<Mapping>* result,
+ std::string* err) {
+ const int fd = open(filename.c_str(), O_RDONLY);
+ if (fd == -1) {
+ *err = strerror(errno);
+ return -errno;
+ }
+
+ // The lexer needs a NUL byte at the end of its input, to know when it's done.
+ // Allocate an entire zero page to be sure.
+ const size_t page_size = sysconf(_SC_PAGESIZE);
+ const size_t file_size = lseek(fd, 0, SEEK_END);
+ const size_t mapping_size = RoundUp(file_size + page_size, page_size);
+ char* map_region = static_cast<char*>(mmap(nullptr, mapping_size,
+ PROT_READ,
+ MAP_PRIVATE, fd, 0));
+ if (map_region == reinterpret_cast<char*>(-1)) {
+ *err = strerror(errno);
+ return -errno;
+ }
+
+ std::unique_ptr<Mapping> mapping(
+ new Mapping(map_region, file_size, mapping_size));
+
+ // Replace the extra page at the end with a read-write anonymous page that
+ // we write an explicit NUL-terminator into.
+ if (mmap(map_region + mapping_size - page_size, page_size,
+ PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) ==
+ reinterpret_cast<char*>(-1)) {
+ *err = std::string("mapping zero page failed: ") + strerror(errno);
+ return -errno;
+ }
+ map_region[mapping_size - page_size] = '\0';
+
+ close(fd);
+ *result = std::move(mapping);
+ return 0;
+}
+#endif // ! _WIN32
+
void SetCloseOnExec(int fd) {
#ifndef _WIN32
int flags = fcntl(fd, F_GETFD);
@@ -622,3 +698,28 @@
}
return true;
}
+
+bool PropagateError(std::string* err,
+ const std::vector<std::string>& subtask_err) {
+ for (const std::string& e : subtask_err) {
+ if (!e.empty()) {
+ *err = e;
+ return false;
+ }
+ }
+ return true;
+}
+
+size_t GetThreadSlotCount() {
+ static size_t result = std::max<size_t>(1, GetProcessorCount());
+ return result;
+}
+
+size_t GetThreadSlotIndex() {
+ static std::atomic<size_t> counter {};
+ thread_local size_t result = []() {
+ size_t result = ++counter;
+ return result % GetThreadSlotCount();
+ }();
+ return result;
+}
diff --git a/src/util.h b/src/util.h
index 6a4a7a9..9ae765a 100644
--- a/src/util.h
+++ b/src/util.h
@@ -21,6 +21,11 @@
#include <stdint.h>
#endif
+#include <limits.h>
+#include <stdarg.h>
+
+#include <atomic>
+#include <memory>
#include <string>
#include <vector>
using namespace std;
@@ -31,7 +36,9 @@
#define NORETURN __attribute__((noreturn))
#endif
-/// Log a fatal message and exit.
+/// Log a fatal message and exit. This function should not be called from a
+/// worker thread, because doing so would tend to produce interleaved output,
+/// and it might destruct static globals that other threads are using.
NORETURN void Fatal(const char* msg, ...);
// Have a generic fall-through for different versions of C/C++.
@@ -50,9 +57,15 @@
/// Log a warning message.
void Warning(const char* msg, ...);
+void Warning(const char* msg, va_list ap);
/// Log an error message.
void Error(const char* msg, ...);
+void Error(const char* msg, va_list ap);
+
+/// Log an informational message.
+void Info(const char* msg, ...);
+void Info(const char* msg, va_list ap);
/// Canonicalize a path like "foo/../bar.h" into just "bar.h".
/// |slash_bits| has bits set starting from lowest for a backslash that was
@@ -73,6 +86,39 @@
/// Returns -errno and fills in \a err on error.
int ReadFile(const string& path, string* contents, string* err);
+#ifndef _WIN32
+class Mapping {
+public:
+ Mapping(char* base, size_t file_size, size_t mapping_size)
+ : base_(base), file_size_(file_size), mapping_size_(mapping_size) {}
+ ~Mapping() { unmap(); }
+ void unmap();
+
+ Mapping(const Mapping&) = delete;
+ Mapping& operator=(const Mapping&) = delete;
+
+ char* base() const { return base_; }
+
+ /// This size does not include the extra NUL at the end.
+ size_t file_size() const { return file_size_; }
+
+ /// The size of the entire mapping, including the extra zero page at the end.
+ size_t mapping_size() const { return mapping_size_; }
+
+private:
+ char* base_ = nullptr;
+ size_t file_size_ = 0;
+ size_t mapping_size_ = 0;
+};
+
+/// Map the file into memory as read-only. The file is followed by a zero page,
+/// and the first byte after the file's content is guaranteed to be NUL. The
+/// lexer relies on this property to efficiently detect the end of input.
+int MapFile(const std::string& filename,
+ std::unique_ptr<Mapping>* result,
+ std::string* err);
+#endif // ! _WIN32
+
/// Mark a file descriptor to not be inherited on exec()s.
void SetCloseOnExec(int fd);
@@ -122,4 +168,93 @@
NORETURN void Win32Fatal(const char* function, const char* hint = NULL);
#endif
+template <typename T, typename U>
+decltype(T() + U()) RoundUp(T x, U y) {
+ return (x + (y - 1)) / y * y;
+}
+
+template <typename T>
+std::vector<std::pair<T, T>> SplitByChunkSize(T total, T chunk_size) {
+ std::vector<std::pair<T, T>> result;
+ result.reserve(total / chunk_size + 1);
+ T start = 0;
+ while (start < total) {
+ T finish = std::min<T>(start + chunk_size, total);
+ result.emplace_back(start, finish);
+ start = finish;
+ }
+ return result;
+}
+
+template <typename T, typename C>
+std::vector<std::pair<T, T>> SplitByCount(T total, C count) {
+ T chunk_size = std::max<T>(1, RoundUp(total, count) / count);
+ return SplitByChunkSize(total, chunk_size);
+}
+
+template <typename T>
+struct IntegralRange {
+ IntegralRange(T start, T stop) : start_(start), stop_(stop) {}
+ size_t size() const { return (stop_ >= start_) ? (stop_ - start_) : 0; }
+ T operator[](size_t idx) const { return start_ + idx; }
+private:
+ T start_;
+ T stop_;
+};
+
+/// Atomically set |*dest| to the lesser of its current value and the given
+/// |candidate| value, using the given less-than comparator.
+template <typename T, typename LessThan=std::less<T>>
+T AtomicUpdateMinimum(std::atomic<T>* dest, T candidate,
+ LessThan less_than=LessThan()) {
+ T current = dest->load();
+ while (true) {
+ if (!less_than(candidate, current)) {
+ return current;
+ }
+ if (dest->compare_exchange_weak(current, candidate)) {
+ return candidate;
+ }
+ }
+}
+
+/// Atomically set |*dest| to the greater of its current value and the given
+/// |candidate| value, using the given less-than comparator.
+template <typename T, typename LessThan=std::less<T>>
+T AtomicUpdateMaximum(std::atomic<T>* dest, T candidate,
+ LessThan less_than=LessThan()) {
+ // The C++ standard library (e.g. std::max) orders values using operator<
+ // rather than operator>, so flip the argument order with a lambda rather than
+ // take a greater-than functor argument.
+ return AtomicUpdateMinimum(dest, candidate, [&](const T& x, const T& y) {
+ return less_than(y, x);
+ });
+}
+
+/// If any string in the given vector is non-empty, copy it to |*err| and return
+/// false. Otherwise return true.
+bool PropagateError(std::string* err,
+ const std::vector<std::string>& subtask_err);
+
+/// Assign each thread a "slot" within a fixed number of slots. The number of
+/// slots, and each thread's slot index, is fixed until the program exits.
+size_t GetThreadSlotCount();
+size_t GetThreadSlotIndex();
+
+/// False sharing between CPU cores can have a measurable effect on ninja
+/// performance. Avoid it by overaligning some frequently-accessed data
+/// structures, which ensures that different copies of a struct are on different
+/// cache lines. A typical L1 cache line size is 64 bytes. Perhaps this code
+/// could eventually use std::hardware_destructive_interference_size from C++17,
+/// but that constant is currently missing from libc++.
+///
+/// This overalignment is only done with C++17 to avoid confusion. In previous
+/// C++ versions, operator new didn't support overalignment. (See WG21 paper
+/// P0035R4.)
+#if __cplusplus >= 201703L
+#define NINJA_ALIGNAS_CACHE_LINE alignas(64)
+#else
+#define NINJA_ALIGNAS_CACHE_LINE
+#endif
+
#endif // NINJA_UTIL_H_
