v1.5.0
diff --git a/.clang-format b/.clang-format
new file mode 100644
index 0000000..1841c03
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,25 @@
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This isn't meant to be authoritative, but it's good enough to be useful.
+# Still use your best judgement for formatting decisions: clang-format
+# sometimes makes strange choices.
+
+BasedOnStyle: Google
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+Cpp11BracedListStyle: false
+IndentCaseLabels: false
diff --git a/.gitignore b/.gitignore
index 501a02d..40a610d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,9 +8,10 @@
/ninja
/build_log_perftest
/canon_perftest
+/depfile_parser_perftest
/hash_collision_bench
/ninja_test
-/parser_perftest
+/manifest_parser_perftest
/graph.png
/doc/manual.html
/doc/doxygen
diff --git a/RELEASING b/RELEASING
index 25926db..c4973b2 100644
--- a/RELEASING
+++ b/RELEASING
@@ -1,17 +1,18 @@
Notes to myself on all the steps to make for a Ninja release.
Push new release branch:
-1. update src/version.cc with new version (with ".git")
-2. git checkout release; git merge master
-3. fix version number in src/version.cc (it will likely conflict in the above)
-4. fix version in doc/manual.asciidoc
-5. commit, tag, push (don't forget to push --tags)
-6. construct release notes from prior notes
+1. Consider sending a heads-up to the ninja-build mailing list first
+2. update src/version.cc with new version (with ".git"), commit to master
+3. git checkout release; git merge master
+4. fix version number in src/version.cc (it will likely conflict in the above)
+5. fix version in doc/manual.asciidoc
+6. commit, tag, push (don't forget to push --tags)
+7. construct release notes from prior notes
credits: git shortlog -s --no-merges REV..
Release on github:
-1. (haven't tried this yet)
- https://github.com/blog/1547-release-your-software
+1. https://github.com/blog/1547-release-your-software
+ Add binaries to https://github.com/martine/ninja/releases
Make announcement on mailing list:
1. copy old mail
diff --git a/bootstrap.py b/bootstrap.py
index 66ec85b..026396b 100755
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -34,10 +34,12 @@
parser.add_option('--x64', action='store_true',
help='force 64-bit build (Windows)',)
parser.add_option('--platform',
- help='target platform (' + '/'.join(platform_helper.platforms()) + ')',
+ help='target platform (' +
+ '/'.join(platform_helper.platforms()) + ')',
choices=platform_helper.platforms())
parser.add_option('--force-pselect', action='store_true',
- help="ppoll() is used by default on Linux, OpenBSD and Bitrig, but older versions might need to use pselect instead",)
+ help='ppoll() is used by default where available, '
+ 'but some platforms might need to use pselect instead',)
(options, conf_args) = parser.parse_args()
@@ -109,7 +111,8 @@
cflags.append('-D_WIN32_WINNT=0x0501')
if options.x64:
cflags.append('-m64')
-if (platform.is_linux() or platform.is_openbsd() or platform.is_bitrig()) and not options.force_pselect:
+if (platform.is_linux() or platform.is_openbsd() or platform.is_bitrig()) and \
+ not options.force_pselect:
cflags.append('-DUSE_PPOLL')
if options.force_pselect:
conf_args.append("--force-pselect")
@@ -153,8 +156,8 @@
Done!
Note: to work around Windows file locking, where you can't rebuild an
-in-use binary, to run ninja after making any changes to build ninja itself
-you should run ninja.bootstrap instead.""")
+in-use binary, to run ninja after making any changes to build ninja
+itself you should run ninja.bootstrap instead.""")
else:
print('Building ninja using itself...')
run([sys.executable, 'configure.py'] + conf_args)
diff --git a/configure.py b/configure.py
index 9fe3be8..64123a0 100755
--- a/configure.py
+++ b/configure.py
@@ -32,10 +32,12 @@
parser = OptionParser()
profilers = ['gmon', 'pprof']
parser.add_option('--platform',
- help='target platform (' + '/'.join(platform_helper.platforms()) + ')',
+ help='target platform (' +
+ '/'.join(platform_helper.platforms()) + ')',
choices=platform_helper.platforms())
parser.add_option('--host',
- help='host platform (' + '/'.join(platform_helper.platforms()) + ')',
+ help='host platform (' +
+ '/'.join(platform_helper.platforms()) + ')',
choices=platform_helper.platforms())
parser.add_option('--debug', action='store_true',
help='enable debugging extras',)
@@ -48,7 +50,8 @@
help='use EXE as the Python interpreter',
default=os.path.basename(sys.executable))
parser.add_option('--force-pselect', action='store_true',
- help="ppoll() is used by default where available, but some platforms may need to use pselect instead",)
+ help='ppoll() is used by default where available, '
+ 'but some platforms may need to use pselect instead',)
(options, args) = parser.parse_args()
if args:
print('ERROR: extra unparsed command-line arguments:', args)
@@ -125,6 +128,8 @@
'/DNOMINMAX', '/D_CRT_SECURE_NO_WARNINGS',
'/D_VARIADIC_MAX=10',
'/DNINJA_PYTHON="%s"' % options.with_python]
+ if platform.msvc_needs_fs():
+ cflags.append('/FS')
ldflags = ['/DEBUG', '/libpath:$builddir']
if not options.debug:
cflags += ['/Ox', '/DNDEBUG', '/GL']
@@ -165,7 +170,8 @@
cflags.append('-fno-omit-frame-pointer')
libs.extend(['-Wl,--no-as-needed', '-lprofiler'])
-if (platform.is_linux() or platform.is_openbsd() or platform.is_bitrig()) and not options.force_pselect:
+if (platform.is_linux() or platform.is_openbsd() or platform.is_bitrig()) and \
+ not options.force_pselect:
cflags.append('-DUSE_PPOLL')
def shell_escape(str):
@@ -322,7 +328,10 @@
gtest_all_incs = '-I%s -I%s' % (path, os.path.join(path, 'include'))
if platform.is_msvc():
- gtest_cflags = '/nologo /EHsc /Zi /D_VARIADIC_MAX=10 ' + gtest_all_incs
+ gtest_cflags = '/nologo /EHsc /Zi /D_VARIADIC_MAX=10 '
+ if platform.msvc_needs_fs():
+ gtest_cflags += '/FS '
+ gtest_cflags += gtest_all_incs
else:
gtest_cflags = '-fvisibility=hidden ' + gtest_all_incs
objs += n.build(built('gtest-all' + objext), 'cxx',
@@ -356,7 +365,7 @@
objs += cxx(name, variables=[('cflags', '$test_cflags')])
if platform.is_windows():
for name in ['includes_normalize_test', 'msvc_helper_test']:
- objs += cxx(name, variables=[('cflags', test_cflags)])
+ objs += cxx(name, variables=[('cflags', '$test_cflags')])
if not platform.is_windows():
test_libs.append('-lpthread')
@@ -368,18 +377,21 @@
n.comment('Ancillary executables.')
-objs = cxx('parser_perftest')
-all_targets += n.build(binary('parser_perftest'), 'link', objs,
- implicit=ninja_lib, variables=[('libs', libs)])
objs = cxx('build_log_perftest')
all_targets += n.build(binary('build_log_perftest'), 'link', objs,
implicit=ninja_lib, variables=[('libs', libs)])
objs = cxx('canon_perftest')
all_targets += n.build(binary('canon_perftest'), 'link', objs,
implicit=ninja_lib, variables=[('libs', libs)])
+objs = cxx('depfile_parser_perftest')
+all_targets += n.build(binary('depfile_parser_perftest'), 'link', objs,
+ implicit=ninja_lib, variables=[('libs', libs)])
objs = cxx('hash_collision_bench')
all_targets += n.build(binary('hash_collision_bench'), 'link', objs,
implicit=ninja_lib, variables=[('libs', libs)])
+objs = cxx('manifest_parser_perftest')
+all_targets += n.build(binary('manifest_parser_perftest'), 'link', objs,
+ implicit=ninja_lib, variables=[('libs', libs)])
n.newline()
n.comment('Generate a graph using the "graph" tool.')
diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc
index 6b2296f..bacb5f6 100644
--- a/doc/manual.asciidoc
+++ b/doc/manual.asciidoc
@@ -1,7 +1,7 @@
Ninja
=====
Evan Martin <martine@danga.com>
-v1.4.0, September 2013
+v1.5.0, June 2014
Introduction
@@ -581,9 +581,13 @@
http://msdn.microsoft.com/en-us/library/hdkef6tk(v=vs.90).aspx[`/showIncludes`
flag]. Briefly, this means the tool outputs specially-formatted lines
to its stdout. Ninja then filters these lines from the displayed
- output. No `depfile` attribute is necessary.
+ output. No `depfile` attribute is necessary, but the localized string
+ in front of the the header file path. For instance
+ `msvc_deps_prefix = Note: including file: `
+ for a English Visual Studio (the default). Should be globally defined.
+
----
+msvc_deps_prefix = Note: including file:
rule cc
deps = msvc
command = cl /showIncludes -c $in /Fo$out
@@ -645,6 +649,21 @@
----------------
+The `console` pool
+^^^^^^^^^^^^^^^^^^
+
+_Available since Ninja 1.5._
+
+There exists a pre-defined pool named `console` with a depth of 1. It has
+the special property that any task in the pool has direct access to the
+standard input, output and error streams provided to Ninja, which are
+normally connected to the user's console (hence the name) but could be
+redirected. This can be useful for interactive tasks or long-running tasks
+which produce status updates on the console (such as test suites).
+
+While a task in the `console` pool is running, Ninja's regular output (such
+as progress status and output from concurrent tasks) is buffered until
+it completes.
Ninja file reference
--------------------
@@ -773,6 +792,10 @@
stored as `.ninja_deps` in the `builddir`, see <<ref_toplevel,the
discussion of `builddir`>>.
+`msvc_deps_prefix`:: _(Available since Ninja 1.5.)_ defines the string
+ which should be stripped from msvc's /showIncludes output. Only
+ needed when `deps = msvc` and no English Visual Studio version is used.
+
`description`:: a short description of the command, used to pretty-print
the command as it's running. The `-v` flag controls whether to print
the full command or its description; if a command fails, the full command
@@ -784,9 +807,9 @@
rebuilt if the command line changes; and secondly, they are not
cleaned by default.
-`in`:: the shell-quoted space-separated list of files provided as
- inputs to the build line referencing this `rule`. (`$in` is provided
- solely for convenience; if you need some subset or variant of this
+`in`:: the space-separated list of files provided as inputs to the build line
+ referencing this `rule`, shell-quoted if it appears in commands. (`$in` is
+ provided solely for convenience; if you need some subset or variant of this
list of files, just construct a new variable with that list and use
that instead.)
@@ -795,8 +818,8 @@
`$rspfile_content`; this works around a bug in the MSVC linker where
it uses a fixed-size buffer for processing input.)
-`out`:: the shell-quoted space-separated list of files provided as
- outputs to the build line referencing this `rule`.
+`out`:: the space-separated list of files provided as outputs to the build line
+ referencing this `rule`, shell-quoted if it appears in commands.
`restat`:: if present, causes Ninja to re-stat the command's outputs
after execution of the command. Each output whose modification time
diff --git a/misc/bash-completion b/misc/bash-completion
index 2d6975b..6edf4df 100644
--- a/misc/bash-completion
+++ b/misc/bash-completion
@@ -16,25 +16,43 @@
# . path/to/ninja/misc/bash-completion
_ninja_target() {
- local cur targets dir line targets_command OPTIND
- cur="${COMP_WORDS[COMP_CWORD]}"
+ local cur prev targets dir line targets_command OPTIND
- if [[ "$cur" == "--"* ]]; then
- # there is currently only one argument that takes --
- COMPREPLY=($(compgen -P '--' -W 'version' -- "${cur:2}"))
- else
- dir="."
- line=$(echo ${COMP_LINE} | cut -d" " -f 2-)
- # filter out all non relevant arguments but keep C for dirs
- while getopts C:f:j:l:k:nvd:t: opt "${line[@]}"; do
- case $opt in
- C) dir="$OPTARG" ;;
- esac
- done;
- targets_command="ninja -C ${dir} -t targets all"
- targets=$((${targets_command} 2>/dev/null) | awk -F: '{print $1}')
- COMPREPLY=($(compgen -W "$targets" -- "$cur"))
- fi
+ # When available, use bash_completion to:
+ # 1) Complete words when the cursor is in the middle of the word
+ # 2) Complete paths with files or directories, as appropriate
+ if _get_comp_words_by_ref cur prev &>/dev/null ; then
+ case $prev in
+ -f)
+ _filedir
+ return 0
+ ;;
+ -C)
+ _filedir -d
+ return 0
+ ;;
+ esac
+ else
+ cur="${COMP_WORDS[COMP_CWORD]}"
+ fi
+
+ if [[ "$cur" == "--"* ]]; then
+ # there is currently only one argument that takes --
+ COMPREPLY=($(compgen -P '--' -W 'version' -- "${cur:2}"))
+ else
+ dir="."
+ line=$(echo ${COMP_LINE} | cut -d" " -f 2-)
+ # filter out all non relevant arguments but keep C for dirs
+ while getopts :C:f:j:l:k:nvd:t: opt $line; do
+ case $opt in
+ # eval for tilde expansion
+ C) eval dir="$OPTARG" ;;
+ esac
+ done;
+ targets_command="eval ninja -C \"${dir}\" -t targets all"
+ targets=$((${targets_command} 2>/dev/null) | awk -F: '{print $1}')
+ COMPREPLY=($(compgen -W "$targets" -- "$cur"))
+ fi
return
}
complete -F _ninja_target ninja
diff --git a/misc/ninja-mode.el b/misc/ninja-mode.el
index d939206..36ada6f 100644
--- a/misc/ninja-mode.el
+++ b/misc/ninja-mode.el
@@ -1,3 +1,5 @@
+;;; ninja-mode.el --- Major mode for editing .ninja files
+
;; Copyright 2011 Google Inc. All Rights Reserved.
;;
;; Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,10 +14,14 @@
;; See the License for the specific language governing permissions and
;; limitations under the License.
+;;; Commentary:
+
;; Simple emacs mode for editing .ninja files.
;; Just some syntax highlighting for now.
-(setq ninja-keywords
+;;; Code:
+
+(defvar ninja-keywords
(list
'("^#.*" . font-lock-comment-face)
(cons (concat "^" (regexp-opt '("rule" "build" "subninja" "include"
@@ -26,8 +32,10 @@
;; Variable expansion.
'("\\($[[:alnum:]_]+\\)" . (1 font-lock-variable-name-face))
;; Rule names
- '("rule \\([[:alnum:]_]+\\)" . (1 font-lock-function-name-face))
+ '("rule \\([[:alnum:]_-]+\\)" . (1 font-lock-function-name-face))
))
+
+;;;###autoload
(define-derived-mode ninja-mode fundamental-mode "ninja"
(setq comment-start "#")
; Pass extra "t" to turn off syntax-based fontification -- we don't want
@@ -35,8 +43,10 @@
(setq font-lock-defaults '(ninja-keywords t))
)
-(provide 'ninja-mode)
-
;; Run ninja-mode for files ending in .ninja.
;;;###autoload
(add-to-list 'auto-mode-alist '("\\.ninja$" . ninja-mode))
+
+(provide 'ninja-mode)
+
+;;; ninja-mode.el ends here
diff --git a/misc/ninja.vim b/misc/ninja.vim
index d813267..f34588f 100644
--- a/misc/ninja.vim
+++ b/misc/ninja.vim
@@ -1,10 +1,10 @@
" ninja build file syntax.
" Language: ninja build file as described at
" http://martine.github.com/ninja/manual.html
-" Version: 1.3
-" Last Change: 2013/04/16
+" Version: 1.4
+" Last Change: 2014/05/13
" Maintainer: Nicolas Weber <nicolasweber@gmx.de>
-" Version 1.3 of this script is in the upstream vim repository and will be
+" Version 1.4 of this script is in the upstream vim repository and will be
" included in the next vim release. If you change this, please send your change
" upstream.
@@ -55,6 +55,7 @@
" $simple_varname -> variable
" ${varname} -> variable
+syn match ninjaDollar "\$\$"
syn match ninjaWrapLineOperator "\$$"
syn match ninjaSimpleVar "\$[a-zA-Z0-9_-]\+"
syn match ninjaVar "\${[a-zA-Z0-9_.-]\+}"
@@ -70,6 +71,7 @@
hi def link ninjaKeyword Keyword
hi def link ninjaRuleCommand Statement
hi def link ninjaPoolCommand Statement
+hi def link ninjaDollar ninjaOperator
hi def link ninjaWrapLineOperator ninjaOperator
hi def link ninjaOperator Operator
hi def link ninjaSimpleVar ninjaVar
diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py
index d69e3e4..14b932f 100644
--- a/misc/ninja_syntax.py
+++ b/misc/ninja_syntax.py
@@ -8,10 +8,9 @@
"""
import textwrap
-import re
def escape_path(word):
- return word.replace('$ ','$$ ').replace(' ','$ ').replace(':', '$:')
+ return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:')
class Writer(object):
def __init__(self, output, width=78):
@@ -61,21 +60,20 @@
def build(self, outputs, rule, inputs=None, implicit=None, order_only=None,
variables=None):
outputs = self._as_list(outputs)
- all_inputs = self._as_list(inputs)[:]
- out_outputs = list(map(escape_path, outputs))
- all_inputs = list(map(escape_path, all_inputs))
+ out_outputs = [escape_path(x) for x in outputs]
+ all_inputs = [escape_path(x) for x in self._as_list(inputs)]
if implicit:
- implicit = map(escape_path, self._as_list(implicit))
+ implicit = [escape_path(x) for x in self._as_list(implicit)]
all_inputs.append('|')
all_inputs.extend(implicit)
if order_only:
- order_only = map(escape_path, self._as_list(order_only))
+ order_only = [escape_path(x) for x in self._as_list(order_only)]
all_inputs.append('||')
all_inputs.extend(order_only)
self._line('build %s: %s' % (' '.join(out_outputs),
- ' '.join([rule] + all_inputs)))
+ ' '.join([rule] + all_inputs)))
if variables:
if isinstance(variables, dict):
@@ -98,13 +96,13 @@
self._line('default %s' % ' '.join(self._as_list(paths)))
def _count_dollars_before_index(self, s, i):
- """Returns the number of '$' characters right in front of s[i]."""
- dollar_count = 0
- dollar_index = i - 1
- while dollar_index > 0 and s[dollar_index] == '$':
- dollar_count += 1
- dollar_index -= 1
- return dollar_count
+ """Returns the number of '$' characters right in front of s[i]."""
+ dollar_count = 0
+ dollar_index = i - 1
+ while dollar_index > 0 and s[dollar_index] == '$':
+ dollar_count += 1
+ dollar_index -= 1
+ return dollar_count
def _line(self, text, indent=0):
"""Write 'text' word-wrapped at self.width characters."""
@@ -117,19 +115,19 @@
available_space = self.width - len(leading_space) - len(' $')
space = available_space
while True:
- space = text.rfind(' ', 0, space)
- if space < 0 or \
- self._count_dollars_before_index(text, space) % 2 == 0:
- break
+ space = text.rfind(' ', 0, space)
+ if (space < 0 or
+ self._count_dollars_before_index(text, space) % 2 == 0):
+ break
if space < 0:
# No such space; just use the first unescaped space we can find.
space = available_space - 1
while True:
- space = text.find(' ', space + 1)
- if space < 0 or \
- self._count_dollars_before_index(text, space) % 2 == 0:
- break
+ space = text.find(' ', space + 1)
+ if (space < 0 or
+ self._count_dollars_before_index(text, space) % 2 == 0):
+ break
if space < 0:
# Give up on breaking.
break
diff --git a/misc/write_fake_manifests.py b/misc/write_fake_manifests.py
new file mode 100644
index 0000000..837007e
--- /dev/null
+++ b/misc/write_fake_manifests.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+
+"""Writes large manifest files, for manifest parser performance testing.
+
+The generated manifest files are (eerily) similar in appearance and size to the
+ones used in the Chromium project.
+
+Usage:
+ python misc/write_fake_manifests.py outdir # Will run for about 5s.
+
+The program contains a hardcoded random seed, so it will generate the same
+output every time it runs. By changing the seed, it's easy to generate many
+different sets of manifest files.
+"""
+
+import argparse
+import contextlib
+import os
+import random
+import sys
+
+import ninja_syntax
+
+
+def paretoint(avg, alpha):
+ """Returns a random integer that's avg on average, following a power law.
+ alpha determines the shape of the power curve. alpha has to be larger
+ than 1. The closer alpha is to 1, the higher the variation of the returned
+ numbers."""
+ return int(random.paretovariate(alpha) * avg / (alpha / (alpha - 1)))
+
+
+# Based on http://neugierig.org/software/chromium/class-name-generator.html
+def moar(avg_options, p_suffix):
+ kStart = ['render', 'web', 'browser', 'tab', 'content', 'extension', 'url',
+ 'file', 'sync', 'content', 'http', 'profile']
+ kOption = ['view', 'host', 'holder', 'container', 'impl', 'ref',
+ 'delegate', 'widget', 'proxy', 'stub', 'context',
+ 'manager', 'master', 'watcher', 'service', 'file', 'data',
+ 'resource', 'device', 'info', 'provider', 'internals', 'tracker',
+ 'api', 'layer']
+ kOS = ['win', 'mac', 'aura', 'linux', 'android', 'unittest', 'browsertest']
+ num_options = min(paretoint(avg_options, alpha=4), 5)
+ # The original allows kOption to repeat as long as no consecutive options
+ # repeat. This version doesn't allow any option repetition.
+ name = [random.choice(kStart)] + random.sample(kOption, num_options)
+ if random.random() < p_suffix:
+ name.append(random.choice(kOS))
+ return '_'.join(name)
+
+
+class GenRandom(object):
+ def __init__(self):
+ self.seen_names = set([None])
+ self.seen_defines = set([None])
+
+ def _unique_string(self, seen, avg_options=1.3, p_suffix=0.1):
+ s = None
+ while s in seen:
+ s = moar(avg_options, p_suffix)
+ seen.add(s)
+ return s
+
+ def _n_unique_strings(self, n):
+ seen = set([None])
+ return [self._unique_string(seen, avg_options=3, p_suffix=0.4)
+ for _ in xrange(n)]
+
+ def target_name(self):
+ return self._unique_string(p_suffix=0, seen=self.seen_names)
+
+ def path(self):
+ return os.path.sep.join([
+ self._unique_string(self.seen_names, avg_options=1, p_suffix=0)
+ for _ in xrange(1 + paretoint(0.6, alpha=4))])
+
+ def src_obj_pairs(self, path, name):
+ num_sources = paretoint(55, alpha=2) + 1
+ return [(os.path.join('..', '..', path, s + '.cc'),
+ os.path.join('obj', path, '%s.%s.o' % (name, s)))
+ for s in self._n_unique_strings(num_sources)]
+
+ def defines(self):
+ return [
+ '-DENABLE_' + self._unique_string(self.seen_defines).upper()
+ for _ in xrange(paretoint(20, alpha=3))]
+
+
+LIB, EXE = 0, 1
+class Target(object):
+ def __init__(self, gen, kind):
+ self.name = gen.target_name()
+ self.dir_path = gen.path()
+ self.ninja_file_path = os.path.join(
+ 'obj', self.dir_path, self.name + '.ninja')
+ self.src_obj_pairs = gen.src_obj_pairs(self.dir_path, self.name)
+ if kind == LIB:
+ self.output = os.path.join('lib' + self.name + '.a')
+ elif kind == EXE:
+ self.output = os.path.join(self.name)
+ self.defines = gen.defines()
+ self.deps = []
+ self.kind = kind
+ self.has_compile_depends = random.random() < 0.4
+
+ @property
+ def includes(self):
+ return ['-I' + dep.dir_path for dep in self.deps]
+
+
+def write_target_ninja(ninja, target):
+ compile_depends = None
+ if target.has_compile_depends:
+ compile_depends = os.path.join(
+ 'obj', target.dir_path, target.name + '.stamp')
+ ninja.build(compile_depends, 'stamp', target.src_obj_pairs[0][0])
+ ninja.newline()
+
+ ninja.variable('defines', target.defines)
+ if target.deps:
+ ninja.variable('includes', target.includes)
+ ninja.variable('cflags', ['-Wall', '-fno-rtti', '-fno-exceptions'])
+ ninja.newline()
+
+ for src, obj in target.src_obj_pairs:
+ ninja.build(obj, 'cxx', src, implicit=compile_depends)
+ ninja.newline()
+
+ deps = [dep.output for dep in target.deps]
+ libs = [dep.output for dep in target.deps if dep.kind == LIB]
+ if target.kind == EXE:
+ ninja.variable('ldflags', '-Wl,pie')
+ ninja.variable('libs', libs)
+ link = { LIB: 'alink', EXE: 'link'}[target.kind]
+ ninja.build(target.output, link, [obj for _, obj in target.src_obj_pairs],
+ implicit=deps)
+
+
+def write_master_ninja(master_ninja, targets):
+ """Writes master build.ninja file, referencing all given subninjas."""
+ master_ninja.variable('cxx', 'c++')
+ master_ninja.variable('ld', '$cxx')
+ master_ninja.newline()
+
+ master_ninja.pool('link_pool', depth=4)
+ master_ninja.newline()
+
+ master_ninja.rule('cxx', description='CXX $out',
+ command='$cxx -MMD -MF $out.d $defines $includes $cflags -c $in -o $out',
+ depfile='$out.d', deps='gcc')
+ master_ninja.rule('alink', description='LIBTOOL-STATIC $out',
+ command='rm -f $out && libtool -static -o $out $in')
+ master_ninja.rule('link', description='LINK $out', pool='link_pool',
+ command='$ld $ldflags -o $out $in $libs')
+ master_ninja.rule('stamp', description='STAMP $out', command='touch $out')
+ master_ninja.newline()
+
+ for target in targets:
+ master_ninja.subninja(target.ninja_file_path)
+ master_ninja.newline()
+
+ master_ninja.comment('Short names for targets.')
+ for target in targets:
+ if target.name != target.output:
+ master_ninja.build(target.name, 'phony', target.output)
+ master_ninja.newline()
+
+ master_ninja.build('all', 'phony', [target.output for target in targets])
+ master_ninja.default('all')
+
+
+@contextlib.contextmanager
+def FileWriter(path):
+ """Context manager for a ninja_syntax object writing to a file."""
+ try:
+ os.makedirs(os.path.dirname(path))
+ except OSError:
+ pass
+ f = open(path, 'w')
+ yield ninja_syntax.Writer(f)
+ f.close()
+
+
+def random_targets():
+ num_targets = 800
+ gen = GenRandom()
+
+ # N-1 static libraries, and 1 executable depending on all of them.
+ targets = [Target(gen, LIB) for i in xrange(num_targets - 1)]
+ for i in range(len(targets)):
+ targets[i].deps = [t for t in targets[0:i] if random.random() < 0.05]
+
+ last_target = Target(gen, EXE)
+ last_target.deps = targets[:]
+ last_target.src_obj_pairs = last_target.src_obj_pairs[0:10] # Trim.
+ targets.append(last_target)
+ return targets
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('outdir', help='output directory')
+ args = parser.parse_args()
+ root_dir = args.outdir
+
+ random.seed(12345)
+
+ targets = random_targets()
+ for target in targets:
+ with FileWriter(os.path.join(root_dir, target.ninja_file_path)) as n:
+ write_target_ninja(n, target)
+
+ with FileWriter(os.path.join(root_dir, 'build.ninja')) as master_ninja:
+ master_ninja.width = 120
+ write_master_ninja(master_ninja, targets)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/misc/zsh-completion b/misc/zsh-completion
index cd0edfb..2fe16fb 100644
--- a/misc/zsh-completion
+++ b/misc/zsh-completion
@@ -1,3 +1,4 @@
+#compdef ninja
# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +16,47 @@
# Add the following to your .zshrc to tab-complete ninja targets
# . path/to/ninja/misc/zsh-completion
-_ninja() {
- reply=(`(ninja -t targets all 2&>/dev/null) | awk -F: '{print $1}'`)
+__get_targets() {
+ ninja -t targets 2>/dev/null | while read -r a b; do echo $a | cut -d ':' -f1; done;
}
-compctl -K _ninja ninja
+
+__get_tools() {
+ ninja -t list 2>/dev/null | while read -r a b; do echo $a; done | tail -n +2
+}
+
+__get_modes() {
+ ninja -d list 2>/dev/null | while read -r a b; do echo $a; done | tail -n +2 | head -n -1
+}
+
+__modes() {
+ local -a modes
+ modes=(${(fo)"$(__get_modes)"})
+ _describe 'modes' modes
+}
+
+__tools() {
+ local -a tools
+ tools=(${(fo)"$(__get_tools)"})
+ _describe 'tools' tools
+}
+
+__targets() {
+ local -a targets
+ targets=(${(fo)"$(__get_targets)"})
+ _describe 'targets' targets
+}
+
+_arguments \
+ {-h,--help}'[Show help]' \
+ '--version[Print ninja version]' \
+ '-C+[Change to directory before doing anything else]:directories:_directories' \
+ '-f+[Specify input build file (default=build.ninja)]:files:_files' \
+ '-j+[Run N jobs in parallel (default=number of CPUs available)]:number of jobs' \
+ '-l+[Do not start new jobs if the load average is greater than N]:number of jobs' \
+ '-k+[Keep going until N jobs fail (default=1)]:number of jobs' \
+ '-n[Dry run (do not run commands but act like they succeeded)]' \
+ '-v[Show all command lines while building]' \
+ '-d+[Enable debugging (use -d list to list modes)]:modes:__modes' \
+ '-t+[Run a subtool (use -t list to list subtools)]:tools:__tools' \
+ '*::targets:__targets'
+
diff --git a/platform_helper.py b/platform_helper.py
index b7447a1..bc3a125 100644
--- a/platform_helper.py
+++ b/platform_helper.py
@@ -19,10 +19,10 @@
def platforms():
return ['linux', 'darwin', 'freebsd', 'openbsd', 'solaris', 'sunos5',
- 'mingw', 'msvc', 'gnukfreebsd8', 'bitrig']
+ 'mingw', 'msvc', 'gnukfreebsd', 'bitrig']
-class Platform( object ):
- def __init__( self, platform):
+class Platform(object):
+ def __init__(self, platform):
self._platform = platform
if not self._platform is None:
return
@@ -31,7 +31,7 @@
self._platform = 'linux'
elif self._platform.startswith('freebsd'):
self._platform = 'freebsd'
- elif self._platform.startswith('gnukfreebsd8'):
+ elif self._platform.startswith('gnukfreebsd'):
self._platform = 'freebsd'
elif self._platform.startswith('openbsd'):
self._platform = 'openbsd'
@@ -56,6 +56,14 @@
def is_msvc(self):
return self._platform == 'msvc'
+ def msvc_needs_fs(self):
+ import subprocess
+ popen = subprocess.Popen(['cl', '/nologo', '/?'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = popen.communicate()
+ return '/FS ' in str(out)
+
def is_windows(self):
return self.is_mingw() or self.is_msvc()
diff --git a/src/build.cc b/src/build.cc
index 9718f85..64bcea3 100644
--- a/src/build.cc
+++ b/src/build.cc
@@ -97,6 +97,9 @@
++started_edges_;
PrintStatus(edge);
+
+ if (edge->use_console())
+ printer_.SetConsoleLocked(true);
}
void BuildStatus::BuildEdgeFinished(Edge* edge,
@@ -112,10 +115,13 @@
*end_time = (int)(now - start_time_millis_);
running_edges_.erase(i);
+ if (edge->use_console())
+ printer_.SetConsoleLocked(false);
+
if (config_.verbosity == BuildConfig::QUIET)
return;
- if (printer_.is_smart_terminal())
+ if (!edge->use_console() && printer_.is_smart_terminal())
PrintStatus(edge);
// Print the command that is spewing before printing its output.
@@ -145,6 +151,7 @@
}
void BuildStatus::BuildFinished() {
+ printer_.SetConsoleLocked(false);
printer_.PrintOnNewLine("");
}
@@ -488,7 +495,7 @@
bool RealCommandRunner::StartCommand(Edge* edge) {
string command = edge->EvaluateCommand();
- Subprocess* subproc = subprocs_.Add(command);
+ Subprocess* subproc = subprocs_.Add(command, edge->use_console());
if (!subproc)
return false;
subproc_to_edge_.insert(make_pair(subproc, edge));
@@ -534,7 +541,7 @@
for (vector<Edge*>::iterator i = active_edges.begin();
i != active_edges.end(); ++i) {
- string depfile = (*i)->GetBinding("depfile");
+ string depfile = (*i)->GetUnescapedDepfile();
for (vector<Node*>::iterator ni = (*i)->outputs_.begin();
ni != (*i)->outputs_.end(); ++ni) {
// Only delete this output if it was actually modified. This is
@@ -610,6 +617,7 @@
if (failures_allowed && command_runner_->CanRunMore()) {
if (Edge* edge = plan_.FindWork()) {
if (!StartEdge(edge, err)) {
+ Cleanup();
status_->BuildFinished();
return false;
}
@@ -630,6 +638,7 @@
CommandRunner::Result result;
if (!command_runner_->WaitForCommand(&result) ||
result.status == ExitInterrupted) {
+ Cleanup();
status_->BuildFinished();
*err = "interrupted by user";
return false;
@@ -637,6 +646,7 @@
--pending_commands;
if (!FinishCommand(&result, err)) {
+ Cleanup();
status_->BuildFinished();
return false;
}
@@ -686,7 +696,7 @@
// Create response file, if needed
// XXX: this may also block; do we care?
- string rspfile = edge->GetBinding("rspfile");
+ string rspfile = edge->GetUnescapedRspfile();
if (!rspfile.empty()) {
string content = edge->GetBinding("rspfile_content");
if (!disk_interface_->WriteFile(rspfile, content))
@@ -714,9 +724,11 @@
// build perspective.
vector<Node*> deps_nodes;
string deps_type = edge->GetBinding("deps");
+ const string deps_prefix = edge->GetBinding("msvc_deps_prefix");
if (!deps_type.empty()) {
string extract_err;
- if (!ExtractDeps(result, deps_type, &deps_nodes, &extract_err) &&
+ if (!ExtractDeps(result, deps_type, deps_prefix, &deps_nodes,
+ &extract_err) &&
result->success()) {
if (!result->output.empty())
result->output.append("\n");
@@ -760,7 +772,7 @@
restat_mtime = input_mtime;
}
- string depfile = edge->GetBinding("depfile");
+ string depfile = edge->GetUnescapedDepfile();
if (restat_mtime != 0 && deps_type.empty() && !depfile.empty()) {
TimeStamp depfile_mtime = disk_interface_->Stat(depfile);
if (depfile_mtime > restat_mtime)
@@ -776,7 +788,7 @@
plan_.EdgeFinished(edge);
// Delete any left over response file.
- string rspfile = edge->GetBinding("rspfile");
+ string rspfile = edge->GetUnescapedRspfile();
if (!rspfile.empty() && !g_keep_rsp)
disk_interface_->RemoveFile(rspfile);
@@ -802,12 +814,13 @@
bool Builder::ExtractDeps(CommandRunner::Result* result,
const string& deps_type,
+ const string& deps_prefix,
vector<Node*>* deps_nodes,
string* err) {
#ifdef _WIN32
if (deps_type == "msvc") {
CLParser parser;
- result->output = parser.Parse(result->output);
+ result->output = parser.Parse(result->output, deps_prefix);
for (set<string>::iterator i = parser.includes_.begin();
i != parser.includes_.end(); ++i) {
deps_nodes->push_back(state_->GetNode(*i));
@@ -815,7 +828,7 @@
} else
#endif
if (deps_type == "gcc") {
- string depfile = result->edge->GetBinding("depfile");
+ string depfile = result->edge->GetUnescapedDepfile();
if (depfile.empty()) {
*err = string("edge with deps=gcc but no depfile makes no sense");
return false;
diff --git a/src/build.h b/src/build.h
index 5b6c83c..eb3636a 100644
--- a/src/build.h
+++ b/src/build.h
@@ -180,8 +180,9 @@
BuildStatus* status_;
private:
- bool ExtractDeps(CommandRunner::Result* result, const string& deps_type,
- vector<Node*>* deps_nodes, string* err);
+ bool ExtractDeps(CommandRunner::Result* result, const string& deps_type,
+ const string& deps_prefix, vector<Node*>* deps_nodes,
+ string* err);
DiskInterface* disk_interface_;
DependencyScan scan_;
diff --git a/src/build_log.cc b/src/build_log.cc
index b92a06f..3f24c16 100644
--- a/src/build_log.cc
+++ b/src/build_log.cc
@@ -108,9 +108,10 @@
Close();
}
-bool BuildLog::OpenForWrite(const string& path, string* err) {
+bool BuildLog::OpenForWrite(const string& path, const BuildLogUser& user,
+ string* err) {
if (needs_recompaction_) {
- if (!Recompact(path, err))
+ if (!Recompact(path, user, err))
return false;
}
@@ -350,7 +351,8 @@
entry.output.c_str(), entry.command_hash) > 0;
}
-bool BuildLog::Recompact(const string& path, string* err) {
+bool BuildLog::Recompact(const string& path, const BuildLogUser& user,
+ string* err) {
METRIC_RECORD(".ninja_log recompact");
printf("Recompacting log...\n");
@@ -368,7 +370,13 @@
return false;
}
+ vector<StringPiece> dead_outputs;
for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
+ if (user.IsPathDead(i->first)) {
+ dead_outputs.push_back(i->first);
+ continue;
+ }
+
if (!WriteEntry(f, *i->second)) {
*err = strerror(errno);
fclose(f);
@@ -376,6 +384,9 @@
}
}
+ for (size_t i = 0; i < dead_outputs.size(); ++i)
+ entries_.erase(dead_outputs[i]);
+
fclose(f);
if (unlink(path.c_str()) < 0) {
*err = strerror(errno);
diff --git a/src/build_log.h b/src/build_log.h
index eeac5b3..fe81a85 100644
--- a/src/build_log.h
+++ b/src/build_log.h
@@ -25,6 +25,13 @@
struct Edge;
+/// Can answer questions about the manifest for the BuildLog.
+struct BuildLogUser {
+ /// Return if a given output no longer part of the build manifest.
+ /// This is only called during recompaction and doesn't have to be fast.
+ virtual bool IsPathDead(StringPiece s) const = 0;
+};
+
/// Store a log of every command ran for every build.
/// It has a few uses:
///
@@ -36,7 +43,7 @@
BuildLog();
~BuildLog();
- bool OpenForWrite(const string& path, string* err);
+ bool OpenForWrite(const string& path, const BuildLogUser& user, string* err);
bool RecordCommand(Edge* edge, int start_time, int end_time,
TimeStamp restat_mtime = 0);
void Close();
@@ -72,7 +79,7 @@
bool WriteEntry(FILE* f, const LogEntry& entry);
/// Rewrite the known log entries, throwing away old data.
- bool Recompact(const string& path, string* err);
+ bool Recompact(const string& path, const BuildLogUser& user, string* err);
typedef ExternalStringHashMap<LogEntry*>::Type Entries;
const Entries& entries() const { return entries_; }
diff --git a/src/build_log_perftest.cc b/src/build_log_perftest.cc
index a09beb8..810c065 100644
--- a/src/build_log_perftest.cc
+++ b/src/build_log_perftest.cc
@@ -28,10 +28,15 @@
const char kTestFilename[] = "BuildLogPerfTest-tempfile";
+struct NoDeadPaths : public BuildLogUser {
+ virtual bool IsPathDead(StringPiece) const { return false; }
+};
+
bool WriteTestData(string* err) {
BuildLog log;
- if (!log.OpenForWrite(kTestFilename, err))
+ NoDeadPaths no_dead_paths;
+ if (!log.OpenForWrite(kTestFilename, no_dead_paths, err))
return false;
/*
diff --git a/src/build_log_test.cc b/src/build_log_test.cc
index 4639bc9..6738c7b 100644
--- a/src/build_log_test.cc
+++ b/src/build_log_test.cc
@@ -30,7 +30,7 @@
const char kTestFilename[] = "BuildLogTest-tempfile";
-struct BuildLogTest : public StateTestWithBuiltinRules {
+struct BuildLogTest : public StateTestWithBuiltinRules, public BuildLogUser {
virtual void SetUp() {
// In case a crashing test left a stale file behind.
unlink(kTestFilename);
@@ -38,6 +38,7 @@
virtual void TearDown() {
unlink(kTestFilename);
}
+ virtual bool IsPathDead(StringPiece s) const { return false; }
};
TEST_F(BuildLogTest, WriteRead) {
@@ -47,7 +48,7 @@
BuildLog log1;
string err;
- EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
+ EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err));
ASSERT_EQ("", err);
log1.RecordCommand(state_.edges_[0], 15, 18);
log1.RecordCommand(state_.edges_[1], 20, 25);
@@ -75,7 +76,7 @@
BuildLog log;
string contents, err;
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, *this, &err));
ASSERT_EQ("", err);
log.Close();
@@ -86,7 +87,7 @@
EXPECT_EQ(kExpectedVersion, contents);
// Opening the file anew shouldn't add a second version string.
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, *this, &err));
ASSERT_EQ("", err);
log.Close();
@@ -122,7 +123,7 @@
BuildLog log1;
string err;
- EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
+ EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err));
ASSERT_EQ("", err);
log1.RecordCommand(state_.edges_[0], 15, 18);
log1.RecordCommand(state_.edges_[1], 20, 25);
@@ -137,7 +138,7 @@
for (off_t size = statbuf.st_size; size > 0; --size) {
BuildLog log2;
string err;
- EXPECT_TRUE(log2.OpenForWrite(kTestFilename, &err));
+ EXPECT_TRUE(log2.OpenForWrite(kTestFilename, *this, &err));
ASSERT_EQ("", err);
log2.RecordCommand(state_.edges_[0], 15, 18);
log2.RecordCommand(state_.edges_[1], 20, 25);
@@ -261,4 +262,44 @@
ASSERT_EQ(22, e2->end_time);
}
+struct BuildLogRecompactTest : public BuildLogTest {
+ virtual bool IsPathDead(StringPiece s) const { return s == "out2"; }
+};
+
+TEST_F(BuildLogRecompactTest, Recompact) {
+ AssertParse(&state_,
+"build out: cat in\n"
+"build out2: cat in\n");
+
+ BuildLog log1;
+ string err;
+ EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err));
+ ASSERT_EQ("", err);
+ // Record the same edge several times, to trigger recompaction
+ // the next time the log is opened.
+ for (int i = 0; i < 200; ++i)
+ log1.RecordCommand(state_.edges_[0], 15, 18 + i);
+ log1.RecordCommand(state_.edges_[1], 21, 22);
+ log1.Close();
+
+ // Load...
+ BuildLog log2;
+ EXPECT_TRUE(log2.Load(kTestFilename, &err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(2u, log2.entries().size());
+ ASSERT_TRUE(log2.LookupByOutput("out"));
+ ASSERT_TRUE(log2.LookupByOutput("out2"));
+ // ...and force a recompaction.
+ EXPECT_TRUE(log2.OpenForWrite(kTestFilename, *this, &err));
+ log2.Close();
+
+ // "out2" is dead, it should've been removed.
+ BuildLog log3;
+ EXPECT_TRUE(log2.Load(kTestFilename, &err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(1u, log2.entries().size());
+ ASSERT_TRUE(log2.LookupByOutput("out"));
+ ASSERT_FALSE(log2.LookupByOutput("out2"));
+}
+
} // anonymous namespace
diff --git a/src/build_test.cc b/src/build_test.cc
index e206cd8..dad69dc 100644
--- a/src/build_test.cc
+++ b/src/build_test.cc
@@ -44,6 +44,8 @@
ASSERT_FALSE(plan_.FindWork());
sort(ret->begin(), ret->end(), CompareEdgesByOutput::cmp);
}
+
+ void TestPoolWithDepthOne(const char *test_case);
};
TEST_F(PlanTest, Basic) {
@@ -197,15 +199,8 @@
ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err);
}
-TEST_F(PlanTest, PoolWithDepthOne) {
- ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
-"pool foobar\n"
-" depth = 1\n"
-"rule poolcat\n"
-" command = cat $in > $out\n"
-" pool = foobar\n"
-"build out1: poolcat in\n"
-"build out2: poolcat in\n"));
+void PlanTest::TestPoolWithDepthOne(const char* test_case) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, test_case));
GetNode("out1")->MarkDirty();
GetNode("out2")->MarkDirty();
string err;
@@ -239,6 +234,26 @@
ASSERT_EQ(0, edge);
}
+TEST_F(PlanTest, PoolWithDepthOne) {
+ TestPoolWithDepthOne(
+"pool foobar\n"
+" depth = 1\n"
+"rule poolcat\n"
+" command = cat $in > $out\n"
+" pool = foobar\n"
+"build out1: poolcat in\n"
+"build out2: poolcat in\n");
+}
+
+TEST_F(PlanTest, ConsolePool) {
+ TestPoolWithDepthOne(
+"rule poolcat\n"
+" command = cat $in > $out\n"
+" pool = console\n"
+"build out1: poolcat in\n"
+"build out2: poolcat in\n");
+}
+
TEST_F(PlanTest, PoolsWithDepthTwo) {
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"pool foobar\n"
@@ -412,7 +427,7 @@
VirtualFileSystem* fs_;
};
-struct BuildTest : public StateTestWithBuiltinRules {
+struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser {
BuildTest() : config_(MakeConfig()), command_runner_(&fs_),
builder_(&state_, config_, NULL, NULL, &fs_),
status_(config_) {
@@ -435,6 +450,8 @@
builder_.command_runner_.release();
}
+ virtual bool IsPathDead(StringPiece s) const { return false; }
+
/// Rebuild target in the 'working tree' (fs_).
/// State of command_runner_ and logs contents (if specified) ARE MODIFIED.
/// Handy to check for NOOP builds, and higher-level rebuild tests.
@@ -469,7 +486,7 @@
BuildLog build_log, *pbuild_log = NULL;
if (log_path) {
ASSERT_TRUE(build_log.Load(log_path, &err));
- ASSERT_TRUE(build_log.OpenForWrite(log_path, &err));
+ ASSERT_TRUE(build_log.OpenForWrite(log_path, *this, &err));
ASSERT_EQ("", err);
pbuild_log = &build_log;
}
@@ -504,6 +521,7 @@
commands_ran_.push_back(edge->EvaluateCommand());
if (edge->rule().name() == "cat" ||
edge->rule().name() == "cat_rsp" ||
+ edge->rule().name() == "cat_rsp_out" ||
edge->rule().name() == "cc" ||
edge->rule().name() == "touch" ||
edge->rule().name() == "touch-interrupt") {
@@ -513,7 +531,8 @@
}
} else if (edge->rule().name() == "true" ||
edge->rule().name() == "fail" ||
- edge->rule().name() == "interrupt") {
+ edge->rule().name() == "interrupt" ||
+ edge->rule().name() == "console") {
// Don't do anything.
} else {
printf("unknown command\n");
@@ -537,6 +556,15 @@
return true;
}
+ if (edge->rule().name() == "console") {
+ if (edge->use_console())
+ result->status = ExitSuccess;
+ else
+ result->status = ExitFailure;
+ last_command_ = NULL;
+ return true;
+ }
+
if (edge->rule().name() == "fail")
result->status = ExitFailure;
else
@@ -748,13 +776,13 @@
string err;
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"rule cc\n command = cc $in\n depfile = $out.d\n"
-"build foo.o: cc foo.c\n"));
+"build fo$ o.o: cc foo.c\n"));
fs_.Create("foo.c", "");
- EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ EXPECT_TRUE(builder_.AddTarget("fo o.o", &err));
ASSERT_EQ("", err);
ASSERT_EQ(1u, fs_.files_read_.size());
- EXPECT_EQ("foo.o.d", fs_.files_read_[0]);
+ EXPECT_EQ("fo o.o.d", fs_.files_read_[0]);
}
TEST_F(BuildTest, DepFileOK) {
@@ -1275,14 +1303,20 @@
" command = cat $rspfile > $out\n"
" rspfile = $rspfile\n"
" rspfile_content = $long_command\n"
+ "rule cat_rsp_out\n"
+ " command = cat $rspfile > $out\n"
+ " rspfile = $out.rsp\n"
+ " rspfile_content = $long_command\n"
"build out1: cat in\n"
"build out2: cat_rsp in\n"
- " rspfile = out2.rsp\n"
+ " rspfile = out 2.rsp\n"
+ " long_command = Some very long command\n"
+ "build out$ 3: cat_rsp_out in\n"
" long_command = Some very long command\n"));
fs_.Create("out1", "");
fs_.Create("out2", "");
- fs_.Create("out3", "");
+ fs_.Create("out 3", "");
fs_.Tick();
@@ -1293,20 +1327,24 @@
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.AddTarget("out2", &err));
ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.AddTarget("out 3", &err));
+ ASSERT_EQ("", err);
size_t files_created = fs_.files_created_.size();
size_t files_removed = fs_.files_removed_.size();
EXPECT_TRUE(builder_.Build(&err));
- ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // cat + cat_rsp
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
- // The RSP file was created
- ASSERT_EQ(files_created + 1, fs_.files_created_.size());
- ASSERT_EQ(1u, fs_.files_created_.count("out2.rsp"));
+ // The RSP files were created
+ ASSERT_EQ(files_created + 2, fs_.files_created_.size());
+ ASSERT_EQ(1u, fs_.files_created_.count("out 2.rsp"));
+ ASSERT_EQ(1u, fs_.files_created_.count("out 3.rsp"));
- // The RSP file was removed
- ASSERT_EQ(files_removed + 1, fs_.files_removed_.size());
- ASSERT_EQ(1u, fs_.files_removed_.count("out2.rsp"));
+ // The RSP files were removed
+ ASSERT_EQ(files_removed + 2, fs_.files_removed_.size());
+ ASSERT_EQ(1u, fs_.files_removed_.count("out 2.rsp"));
+ ASSERT_EQ(1u, fs_.files_removed_.count("out 3.rsp"));
}
// Test that RSP file is created but not removed for commands, which fail
@@ -1777,7 +1815,7 @@
string err;
const char* manifest =
"rule cc\n command = cc $in\n depfile = $out.d\n deps = gcc\n"
- "build foo.o: cc foo.c\n";
+ "build fo$ o.o: cc foo.c\n";
fs_.Create("foo.c", "");
@@ -1792,9 +1830,9 @@
Builder builder(&state, config_, NULL, &deps_log, &fs_);
builder.command_runner_.reset(&command_runner_);
- EXPECT_TRUE(builder.AddTarget("foo.o", &err));
+ EXPECT_TRUE(builder.AddTarget("fo o.o", &err));
ASSERT_EQ("", err);
- fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
+ fs_.Create("fo o.o.d", "fo\\ o.o: blah.h bar.h\n");
EXPECT_TRUE(builder.Build(&err));
EXPECT_EQ("", err);
@@ -1817,10 +1855,10 @@
Edge* edge = state.edges_.back();
state.GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing.
- EXPECT_TRUE(builder.AddTarget("foo.o", &err));
+ EXPECT_TRUE(builder.AddTarget("fo o.o", &err));
ASSERT_EQ("", err);
- // Expect three new edges: one generating foo.o, and two more from
+ // Expect three new edges: one generating fo o.o, and two more from
// loading the depfile.
ASSERT_EQ(3u, state.edges_.size());
// Expect our edge to now have three inputs: foo.c and two headers.
@@ -1909,3 +1947,20 @@
RebuildTarget("out", manifest, "build_log", "ninja_deps2");
ASSERT_EQ(0u, command_runner_.commands_ran_.size());
}
+
+TEST_F(BuildTest, Console) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule console\n"
+" command = console\n"
+" pool = console\n"
+"build cons: console in.txt\n"));
+
+ fs_.Create("in.txt", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("cons", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+}
diff --git a/src/clean.cc b/src/clean.cc
index 5d1974e..98c638c 100644
--- a/src/clean.cc
+++ b/src/clean.cc
@@ -80,11 +80,11 @@
}
void Cleaner::RemoveEdgeFiles(Edge* edge) {
- string depfile = edge->GetBinding("depfile");
+ string depfile = edge->GetUnescapedDepfile();
if (!depfile.empty())
Remove(depfile);
- string rspfile = edge->GetBinding("rspfile");
+ string rspfile = edge->GetUnescapedRspfile();
if (!rspfile.empty())
Remove(rspfile);
}
diff --git a/src/clean_test.cc b/src/clean_test.cc
index 04cff73..5869bbb 100644
--- a/src/clean_test.cc
+++ b/src/clean_test.cc
@@ -286,8 +286,7 @@
" rspfile = $rspfile\n"
" rspfile_content=$in\n"
"build out1: cc in1\n"
-" rspfile = cc1.rsp\n"
-" rspfile_content=$in\n"));
+" rspfile = cc1.rsp\n"));
fs_.Create("out1", "");
fs_.Create("cc1.rsp", "");
@@ -307,10 +306,9 @@
"build out1: cat in1\n"
"build in2: cat_rsp src2\n"
" rspfile=in2.rsp\n"
-" rspfile_content=$in\n"
"build out2: cat_rsp in2\n"
" rspfile=out2.rsp\n"
-" rspfile_content=$in\n"));
+));
fs_.Create("in1", "");
fs_.Create("out1", "");
fs_.Create("in2.rsp", "");
@@ -336,8 +334,6 @@
EXPECT_EQ(0, fs_.Stat("out2"));
EXPECT_EQ(0, fs_.Stat("in2.rsp"));
EXPECT_EQ(0, fs_.Stat("out2.rsp"));
-
- fs_.files_removed_.clear();
}
TEST_F(CleanTest, CleanFailure) {
@@ -372,3 +368,31 @@
EXPECT_EQ(2, cleaner.cleaned_files_count());
EXPECT_NE(0, fs_.Stat("phony"));
}
+
+TEST_F(CleanTest, CleanDepFileAndRspFileWithSpaces) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc_dep\n"
+" command = cc $in > $out\n"
+" depfile = $out.d\n"
+"rule cc_rsp\n"
+" command = cc $in > $out\n"
+" rspfile = $out.rsp\n"
+" rspfile_content = $in\n"
+"build out$ 1: cc_dep in$ 1\n"
+"build out$ 2: cc_rsp in$ 1\n"
+));
+ fs_.Create("out 1", "");
+ fs_.Create("out 2", "");
+ fs_.Create("out 1.d", "");
+ fs_.Create("out 2.rsp", "");
+
+ Cleaner cleaner(&state_, config_, &fs_);
+ EXPECT_EQ(0, cleaner.CleanAll());
+ EXPECT_EQ(4, cleaner.cleaned_files_count());
+ EXPECT_EQ(4u, fs_.files_removed_.size());
+
+ EXPECT_EQ(0, fs_.Stat("out 1"));
+ EXPECT_EQ(0, fs_.Stat("out 2"));
+ EXPECT_EQ(0, fs_.Stat("out 1.d"));
+ EXPECT_EQ(0, fs_.Stat("out 2.rsp"));
+}
diff --git a/src/debug_flags.cc b/src/debug_flags.cc
index 75f1ea5..8065001 100644
--- a/src/debug_flags.cc
+++ b/src/debug_flags.cc
@@ -15,3 +15,5 @@
bool g_explaining = false;
bool g_keep_rsp = false;
+
+bool g_experimental_statcache = true;
diff --git a/src/debug_flags.h b/src/debug_flags.h
index ba3ebf3..7965585 100644
--- a/src/debug_flags.h
+++ b/src/debug_flags.h
@@ -26,4 +26,6 @@
extern bool g_keep_rsp;
+extern bool g_experimental_statcache;
+
#endif // NINJA_EXPLAIN_H_
diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc
index 5a30c6b..4ca3943 100644
--- a/src/depfile_parser.cc
+++ b/src/depfile_parser.cc
@@ -53,10 +53,10 @@
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
- 0, 128, 128, 128, 128, 128, 128, 128,
+ 0, 128, 0, 0, 0, 0, 0, 0,
+ 128, 128, 0, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
- 128, 128, 128, 128, 128, 128, 128, 128,
- 128, 128, 128, 128, 128, 128, 0, 0,
+ 128, 128, 128, 0, 0, 128, 0, 0,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
@@ -64,7 +64,7 @@
0, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
- 128, 128, 128, 0, 0, 0, 128, 0,
+ 128, 128, 128, 128, 0, 128, 128, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
@@ -84,42 +84,59 @@
};
yych = *in;
- if (yych <= '[') {
+ if (yych <= '=') {
if (yych <= '$') {
- if (yych <= 0x00) goto yy7;
- if (yych <= ' ') goto yy9;
- if (yych <= '#') goto yy6;
- goto yy4;
+ if (yych <= ' ') {
+ if (yych <= 0x00) goto yy7;
+ goto yy9;
+ } else {
+ if (yych <= '!') goto yy5;
+ if (yych <= '#') goto yy9;
+ goto yy4;
+ }
} else {
- if (yych <= '=') goto yy6;
- if (yych <= '?') goto yy9;
- if (yych <= 'Z') goto yy6;
- goto yy9;
+ if (yych <= '*') {
+ if (yych <= '\'') goto yy9;
+ if (yych <= ')') goto yy5;
+ goto yy9;
+ } else {
+ if (yych <= ':') goto yy5;
+ if (yych <= '<') goto yy9;
+ goto yy5;
+ }
}
} else {
- if (yych <= '`') {
- if (yych <= '\\') goto yy2;
- if (yych == '_') goto yy6;
- goto yy9;
+ if (yych <= '^') {
+ if (yych <= 'Z') {
+ if (yych <= '?') goto yy9;
+ goto yy5;
+ } else {
+ if (yych != '\\') goto yy9;
+ }
} else {
- if (yych <= 'z') goto yy6;
- if (yych == '~') goto yy6;
- goto yy9;
+ if (yych <= '{') {
+ if (yych == '`') goto yy9;
+ goto yy5;
+ } else {
+ if (yych <= '|') goto yy9;
+ if (yych <= '~') goto yy5;
+ goto yy9;
+ }
}
}
-yy2:
++in;
- if ((yych = *in) <= '#') {
- if (yych <= '\n') {
+ if ((yych = *in) <= '"') {
+ if (yych <= '\f') {
if (yych <= 0x00) goto yy3;
- if (yych <= '\t') goto yy14;
+ if (yych != '\n') goto yy14;
} else {
+ if (yych <= '\r') goto yy3;
if (yych == ' ') goto yy16;
- if (yych <= '"') goto yy14;
- goto yy16;
+ goto yy14;
}
} else {
if (yych <= 'Z') {
+ if (yych <= '#') goto yy16;
if (yych == '*') goto yy16;
goto yy14;
} else {
@@ -135,10 +152,14 @@
break;
}
yy4:
- ++in;
- if ((yych = *in) == '$') goto yy12;
- goto yy11;
+ yych = *++in;
+ if (yych == '$') goto yy12;
+ goto yy3;
yy5:
+ ++in;
+ yych = *in;
+ goto yy11;
+yy6:
{
// Got a span of plain text.
int len = (int)(in - start);
@@ -148,9 +169,6 @@
out += len;
continue;
}
-yy6:
- yych = *++in;
- goto yy11;
yy7:
++in;
{
@@ -166,12 +184,9 @@
if (yybm[0+yych] & 128) {
goto yy10;
}
- goto yy5;
+ goto yy6;
yy12:
++in;
- if (yybm[0+(yych = *in)] & 128) {
- goto yy10;
- }
{
// De-escape dollar character.
*out++ = '$';
@@ -211,7 +226,7 @@
} else if (!out_.str_) {
out_ = StringPiece(filename, len);
} else if (out_ != StringPiece(filename, len)) {
- *err = "depfile has multiple output paths.";
+ *err = "depfile has multiple output paths";
return false;
}
}
diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc
index cf24a09..b59baf0 100644
--- a/src/depfile_parser.in.cc
+++ b/src/depfile_parser.in.cc
@@ -67,13 +67,13 @@
*out++ = '$';
continue;
}
- '\\' [^\000\n] {
+ '\\' [^\000\r\n] {
// Let backslash before other characters through verbatim.
*out++ = '\\';
*out++ = yych;
continue;
}
- [a-zA-Z0-9+,/_:.~()@=-!]+ {
+ [a-zA-Z0-9+,/_:.~()}{@=!-]+ {
// Got a span of plain text.
int len = (int)(in - start);
// Need to shift it over if we're overwriting backslashes.
@@ -108,7 +108,7 @@
} else if (!out_.str_) {
out_ = StringPiece(filename, len);
} else if (out_ != StringPiece(filename, len)) {
- *err = "depfile has multiple output paths.";
+ *err = "depfile has multiple output paths";
return false;
}
}
diff --git a/src/parser_perftest.cc b/src/depfile_parser_perftest.cc
similarity index 100%
rename from src/parser_perftest.cc
rename to src/depfile_parser_perftest.cc
diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc
index 0f6771a..a5f3321 100644
--- a/src/depfile_parser_test.cc
+++ b/src/depfile_parser_test.cc
@@ -58,6 +58,17 @@
EXPECT_EQ(2u, parser_.ins_.size());
}
+TEST_F(DepfileParserTest, CarriageReturnContinuation) {
+ string err;
+ EXPECT_TRUE(Parse(
+"foo.o: \\\r\n"
+" bar.h baz.h\r\n",
+ &err));
+ ASSERT_EQ("", err);
+ EXPECT_EQ("foo.o", parser_.out_.AsString());
+ EXPECT_EQ(2u, parser_.ins_.size());
+}
+
TEST_F(DepfileParserTest, BackSlashes) {
string err;
EXPECT_TRUE(Parse(
@@ -109,16 +120,19 @@
string err;
EXPECT_TRUE(Parse(
"C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \n"
-" en@quot.header~ t+t-x!=1",
+" en@quot.header~ t+t-x!=1 \n"
+" openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif",
&err));
ASSERT_EQ("", err);
EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h",
parser_.out_.AsString());
- ASSERT_EQ(2u, parser_.ins_.size());
+ ASSERT_EQ(3u, parser_.ins_.size());
EXPECT_EQ("en@quot.header~",
parser_.ins_[0].AsString());
EXPECT_EQ("t+t-x!=1",
parser_.ins_[1].AsString());
+ EXPECT_EQ("openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif",
+ parser_.ins_[2].AsString());
}
TEST_F(DepfileParserTest, UnifyMultipleOutputs) {
@@ -136,4 +150,5 @@
// check that multiple different outputs are rejected by the parser
string err;
EXPECT_FALSE(Parse("foo bar: x y z", &err));
+ ASSERT_EQ("depfile has multiple output paths", err);
}
diff --git a/src/deps_log.cc b/src/deps_log.cc
index 4f1214a..61df387 100644
--- a/src/deps_log.cc
+++ b/src/deps_log.cc
@@ -325,6 +325,9 @@
Deps* deps = deps_[old_id];
if (!deps) continue; // If nodes_[old_id] is a leaf, it has no deps.
+ if (!IsDepsEntryLiveFor(nodes_[old_id]))
+ continue;
+
if (!new_log.RecordDeps(nodes_[old_id], deps->mtime,
deps->node_count, deps->nodes)) {
new_log.Close();
@@ -351,6 +354,16 @@
return true;
}
+bool DepsLog::IsDepsEntryLiveFor(Node* node) {
+ // Skip entries that don't have in-edges or whose edges don't have a
+ // "deps" attribute. They were in the deps log from previous builds, but
+ // the the files they were for were removed from the build and their deps
+ // entries are no longer needed.
+ // (Without the check for "deps", a chain of two or more nodes that each
+ // had deps wouldn't be collected in a single recompaction.)
+ return node->in_edge() && !node->in_edge()->GetBinding("deps").empty();
+}
+
bool DepsLog::UpdateDeps(int out_id, Deps* deps) {
if (out_id >= (int)deps_.size())
deps_.resize(out_id + 1);
diff --git a/src/deps_log.h b/src/deps_log.h
index babf828..cec0257 100644
--- a/src/deps_log.h
+++ b/src/deps_log.h
@@ -88,6 +88,14 @@
/// Rewrite the known log entries, throwing away old data.
bool Recompact(const string& path, string* err);
+ /// Returns if the deps entry for a node is still reachable from the manifest.
+ ///
+ /// The deps log can contain deps entries for files that were built in the
+ /// past but are no longer part of the manifest. This function returns if
+ /// this is the case for a given node. This function is slow, don't call
+ /// it from code that runs on every build.
+ bool IsDepsEntryLiveFor(Node* node);
+
/// Used for tests.
const vector<Node*>& nodes() const { return nodes_; }
const vector<Deps*>& deps() const { return deps_; }
diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc
index 4e6cbac..e8e5138 100644
--- a/src/deps_log_test.cc
+++ b/src/deps_log_test.cc
@@ -163,10 +163,18 @@
// Verify that adding the new deps works and can be compacted away.
TEST_F(DepsLogTest, Recompact) {
+ const char kManifest[] =
+"rule cc\n"
+" command = cc\n"
+" deps = gcc\n"
+"build out.o: cc\n"
+"build other_out.o: cc\n";
+
// Write some deps to the file and grab its size.
int file_size;
{
State state;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
DepsLog log;
string err;
ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
@@ -194,6 +202,7 @@
int file_size_2;
{
State state;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
DepsLog log;
string err;
ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
@@ -215,8 +224,10 @@
// Now reload the file, verify the new deps have replaced the old, then
// recompact.
+ int file_size_3;
{
State state;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
DepsLog log;
string err;
ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
@@ -257,9 +268,53 @@
// The file should have shrunk a bit for the smaller deps.
struct stat st;
ASSERT_EQ(0, stat(kTestFilename, &st));
- int file_size_3 = (int)st.st_size;
+ file_size_3 = (int)st.st_size;
ASSERT_LT(file_size_3, file_size_2);
}
+
+ // Now reload the file and recompact with an empty manifest. The previous
+ // entries should be removed.
+ {
+ State state;
+ // Intentionally not parsing kManifest here.
+ DepsLog log;
+ string err;
+ ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
+
+ Node* out = state.GetNode("out.o");
+ DepsLog::Deps* deps = log.GetDeps(out);
+ ASSERT_TRUE(deps);
+ ASSERT_EQ(1, deps->mtime);
+ ASSERT_EQ(1, deps->node_count);
+ ASSERT_EQ("foo.h", deps->nodes[0]->path());
+
+ Node* other_out = state.GetNode("other_out.o");
+ deps = log.GetDeps(other_out);
+ ASSERT_TRUE(deps);
+ ASSERT_EQ(1, deps->mtime);
+ ASSERT_EQ(2, deps->node_count);
+ ASSERT_EQ("foo.h", deps->nodes[0]->path());
+ ASSERT_EQ("baz.h", deps->nodes[1]->path());
+
+ ASSERT_TRUE(log.Recompact(kTestFilename, &err));
+
+ // The previous entries should have been removed.
+ deps = log.GetDeps(out);
+ ASSERT_FALSE(deps);
+
+ deps = log.GetDeps(other_out);
+ ASSERT_FALSE(deps);
+
+ // The .h files pulled in via deps should no longer have ids either.
+ ASSERT_EQ(-1, state.LookupNode("foo.h")->id());
+ ASSERT_EQ(-1, state.LookupNode("baz.h")->id());
+
+ // The file should have shrunk more.
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ int file_size_4 = (int)st.st_size;
+ ASSERT_LT(file_size_4, file_size_3);
+ }
}
// Verify that invalid file headers cause a new build.
diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index 3233144..ae2146e 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -14,6 +14,8 @@
#include "disk_interface.h"
+#include <algorithm>
+
#include <errno.h>
#include <stdio.h>
#include <string.h>
@@ -31,15 +33,16 @@
string DirName(const string& path) {
#ifdef _WIN32
- const char kPathSeparator = '\\';
+ const char kPathSeparators[] = "\\/";
#else
- const char kPathSeparator = '/';
+ const char kPathSeparators[] = "/";
#endif
-
- string::size_type slash_pos = path.rfind(kPathSeparator);
+ string::size_type slash_pos = path.find_last_of(kPathSeparators);
if (slash_pos == string::npos)
return string(); // Nothing to do.
- while (slash_pos > 0 && path[slash_pos - 1] == kPathSeparator)
+ const char* const kEnd = kPathSeparators + strlen(kPathSeparators);
+ while (slash_pos > 0 &&
+ std::find(kPathSeparators, kEnd, path[slash_pos - 1]) != kEnd)
--slash_pos;
return path.substr(0, slash_pos);
}
@@ -52,6 +55,80 @@
#endif
}
+#ifdef _WIN32
+TimeStamp TimeStampFromFileTime(const FILETIME& filetime) {
+ // FILETIME is in 100-nanosecond increments since the Windows epoch.
+ // We don't much care about epoch correctness but we do want the
+ // resulting value to fit in an integer.
+ uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) |
+ ((uint64_t)filetime.dwLowDateTime);
+ mtime /= 1000000000LL / 100; // 100ns -> s.
+ mtime -= 12622770400LL; // 1600 epoch -> 2000 epoch (subtract 400 years).
+ return (TimeStamp)mtime;
+}
+
+TimeStamp StatSingleFile(const string& path, bool quiet) {
+ WIN32_FILE_ATTRIBUTE_DATA attrs;
+ if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) {
+ DWORD err = GetLastError();
+ if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND)
+ return 0;
+ if (!quiet) {
+ Error("GetFileAttributesEx(%s): %s", path.c_str(),
+ GetLastErrorString().c_str());
+ }
+ return -1;
+ }
+ return TimeStampFromFileTime(attrs.ftLastWriteTime);
+}
+
+#pragma warning(push)
+#pragma warning(disable: 4996) // GetVersionExA is deprecated post SDK 8.1.
+bool IsWindows7OrLater() {
+ OSVERSIONINFO version_info = { sizeof(version_info) };
+ if (!GetVersionEx(&version_info))
+ Fatal("GetVersionEx: %s", GetLastErrorString().c_str());
+ return version_info.dwMajorVersion > 6 ||
+ version_info.dwMajorVersion == 6 && version_info.dwMinorVersion >= 1;
+}
+#pragma warning(pop)
+
+bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
+ bool quiet) {
+ // FindExInfoBasic is 30% faster than FindExInfoStandard.
+ static bool can_use_basic_info = IsWindows7OrLater();
+ // This is not in earlier SDKs.
+ const FINDEX_INFO_LEVELS kFindExInfoBasic =
+ static_cast<FINDEX_INFO_LEVELS>(1);
+ FINDEX_INFO_LEVELS level =
+ can_use_basic_info ? kFindExInfoBasic : FindExInfoStandard;
+ WIN32_FIND_DATAA ffd;
+ HANDLE find_handle = FindFirstFileExA((dir + "\\*").c_str(), level, &ffd,
+ FindExSearchNameMatch, NULL, 0);
+
+ if (find_handle == INVALID_HANDLE_VALUE) {
+ DWORD err = GetLastError();
+ if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND)
+ return true;
+ if (!quiet) {
+ Error("FindFirstFileExA(%s): %s", dir.c_str(),
+ GetLastErrorString().c_str());
+ }
+ return false;
+ }
+ do {
+ if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+ continue;
+ string lowername = ffd.cFileName;
+ transform(lowername.begin(), lowername.end(), lowername.begin(), ::tolower);
+ stamps->insert(make_pair(lowername,
+ TimeStampFromFileTime(ffd.ftLastWriteTime)));
+ } while (FindNextFileA(find_handle, &ffd));
+ FindClose(find_handle);
+ return true;
+}
+#endif // _WIN32
+
} // namespace
// DiskInterface ---------------------------------------------------------------
@@ -75,7 +152,7 @@
// RealDiskInterface -----------------------------------------------------------
-TimeStamp RealDiskInterface::Stat(const string& path) {
+TimeStamp RealDiskInterface::Stat(const string& path) const {
#ifdef _WIN32
// MSDN: "Naming Files, Paths, and Namespaces"
// http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
@@ -86,26 +163,25 @@
}
return -1;
}
- WIN32_FILE_ATTRIBUTE_DATA attrs;
- if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) {
- DWORD err = GetLastError();
- if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND)
- return 0;
- if (!quiet_) {
- Error("GetFileAttributesEx(%s): %s", path.c_str(),
- GetLastErrorString().c_str());
+ if (!use_cache_)
+ return StatSingleFile(path, quiet_);
+
+ string dir = DirName(path);
+ string base(path.substr(dir.size() ? dir.size() + 1 : 0));
+
+ transform(dir.begin(), dir.end(), dir.begin(), ::tolower);
+ transform(base.begin(), base.end(), base.begin(), ::tolower);
+
+ Cache::iterator ci = cache_.find(dir);
+ if (ci == cache_.end()) {
+ ci = cache_.insert(make_pair(dir, DirCache())).first;
+ if (!StatAllFilesInDir(dir.empty() ? "." : dir, &ci->second, quiet_)) {
+ cache_.erase(ci);
+ return -1;
}
- return -1;
}
- const FILETIME& filetime = attrs.ftLastWriteTime;
- // FILETIME is in 100-nanosecond increments since the Windows epoch.
- // We don't much care about epoch correctness but we do want the
- // resulting value to fit in an integer.
- uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) |
- ((uint64_t)filetime.dwLowDateTime);
- mtime /= 1000000000LL / 100; // 100ns -> s.
- mtime -= 12622770400LL; // 1600 epoch -> 2000 epoch (subtract 400 years).
- return (TimeStamp)mtime;
+ DirCache::iterator di = ci->second.find(base);
+ return di != ci->second.end() ? di->second : 0;
#else
struct stat st;
if (stat(path.c_str(), &st) < 0) {
@@ -146,6 +222,9 @@
bool RealDiskInterface::MakeDir(const string& path) {
if (::MakeDir(path) < 0) {
+ if (errno == EEXIST) {
+ return true;
+ }
Error("mkdir(%s): %s", path.c_str(), strerror(errno));
return false;
}
@@ -175,3 +254,11 @@
return 0;
}
}
+
+void RealDiskInterface::AllowStatCache(bool allow) {
+#ifdef _WIN32
+ use_cache_ = allow;
+ if (!use_cache_)
+ cache_.clear();
+#endif
+}
diff --git a/src/disk_interface.h b/src/disk_interface.h
index ff1e21c..a13bced 100644
--- a/src/disk_interface.h
+++ b/src/disk_interface.h
@@ -15,6 +15,7 @@
#ifndef NINJA_DISK_INTERFACE_H_
#define NINJA_DISK_INTERFACE_H_
+#include <map>
#include <string>
using namespace std;
@@ -29,7 +30,7 @@
/// stat() a file, returning the mtime, or 0 if missing and -1 on
/// other errors.
- virtual TimeStamp Stat(const string& path) = 0;
+ virtual TimeStamp Stat(const string& path) const = 0;
/// Create a directory, returning false on failure.
virtual bool MakeDir(const string& path) = 0;
@@ -55,9 +56,13 @@
/// Implementation of DiskInterface that actually hits the disk.
struct RealDiskInterface : public DiskInterface {
- RealDiskInterface() : quiet_(false) {}
+ RealDiskInterface() : quiet_(false)
+#ifdef _WIN32
+ , use_cache_(false)
+#endif
+ {}
virtual ~RealDiskInterface() {}
- virtual TimeStamp Stat(const string& path);
+ virtual TimeStamp Stat(const string& path) const;
virtual bool MakeDir(const string& path);
virtual bool WriteFile(const string& path, const string& contents);
virtual string ReadFile(const string& path, string* err);
@@ -65,6 +70,21 @@
/// Whether to print on errors. Used to make a test quieter.
bool quiet_;
+
+ /// Whether stat information can be cached. Only has an effect on Windows.
+ void AllowStatCache(bool allow);
+
+ private:
+#ifdef _WIN32
+ /// Whether stat information can be cached.
+ bool use_cache_;
+
+ typedef map<string, TimeStamp> DirCache;
+ // TODO: Neither a map nor a hashmap seems ideal here. If the statcache
+ // works out, come up with a better data structure.
+ typedef map<string, DirCache> Cache;
+ mutable Cache cache_;
+#endif
};
#endif // NINJA_DISK_INTERFACE_H_
diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc
index 55822a6..f4e0bb0 100644
--- a/src/disk_interface_test.cc
+++ b/src/disk_interface_test.cc
@@ -76,6 +76,33 @@
EXPECT_GT(disk_.Stat("file"), 1);
}
+#ifdef _WIN32
+TEST_F(DiskInterfaceTest, StatCache) {
+ disk_.AllowStatCache(true);
+
+ ASSERT_TRUE(Touch("file1"));
+ ASSERT_TRUE(Touch("fiLE2"));
+ ASSERT_TRUE(disk_.MakeDir("subdir"));
+ ASSERT_TRUE(Touch("subdir\\subfile1"));
+ ASSERT_TRUE(Touch("subdir\\SUBFILE2"));
+ ASSERT_TRUE(Touch("subdir\\SUBFILE3"));
+
+ EXPECT_GT(disk_.Stat("FIle1"), 1);
+ EXPECT_GT(disk_.Stat("file1"), 1);
+
+ EXPECT_GT(disk_.Stat("subdir/subfile2"), 1);
+ EXPECT_GT(disk_.Stat("sUbdir\\suBFile1"), 1);
+
+ // Test error cases.
+ disk_.quiet_ = true;
+ string bad_path("cc:\\foo");
+ EXPECT_EQ(-1, disk_.Stat(bad_path));
+ EXPECT_EQ(-1, disk_.Stat(bad_path));
+ EXPECT_EQ(0, disk_.Stat("nosuchfile"));
+ EXPECT_EQ(0, disk_.Stat("nosuchdir/nosuchfile"));
+}
+#endif
+
TEST_F(DiskInterfaceTest, ReadFile) {
string err;
EXPECT_EQ("", disk_.ReadFile("foobar", &err));
@@ -93,7 +120,18 @@
}
TEST_F(DiskInterfaceTest, MakeDirs) {
- EXPECT_TRUE(disk_.MakeDirs("path/with/double//slash/"));
+ string path = "path/with/double//slash/";
+ EXPECT_TRUE(disk_.MakeDirs(path.c_str()));
+ FILE* f = fopen((path + "a_file").c_str(), "w");
+ EXPECT_TRUE(f);
+ EXPECT_EQ(0, fclose(f));
+#ifdef _WIN32
+ string path2 = "another\\with\\back\\\\slashes\\";
+ EXPECT_TRUE(disk_.MakeDirs(path2.c_str()));
+ FILE* f2 = fopen((path2 + "a_file").c_str(), "w");
+ EXPECT_TRUE(f2);
+ EXPECT_EQ(0, fclose(f2));
+#endif
}
TEST_F(DiskInterfaceTest, RemoveFile) {
@@ -109,7 +147,7 @@
StatTest() : scan_(&state_, NULL, NULL, this) {}
// DiskInterface implementation.
- virtual TimeStamp Stat(const string& path);
+ virtual TimeStamp Stat(const string& path) const;
virtual bool WriteFile(const string& path, const string& contents) {
assert(false);
return true;
@@ -129,12 +167,12 @@
DependencyScan scan_;
map<string, TimeStamp> mtimes_;
- vector<string> stats_;
+ mutable vector<string> stats_;
};
-TimeStamp StatTest::Stat(const string& path) {
+TimeStamp StatTest::Stat(const string& path) const {
stats_.push_back(path);
- map<string, TimeStamp>::iterator i = mtimes_.find(path);
+ map<string, TimeStamp>::const_iterator i = mtimes_.find(path);
if (i == mtimes_.end())
return 0; // File not found.
return i->second;
diff --git a/src/edit_distance.cc b/src/edit_distance.cc
index cc4483f..9553c6e 100644
--- a/src/edit_distance.cc
+++ b/src/edit_distance.cc
@@ -14,6 +14,7 @@
#include "edit_distance.h"
+#include <algorithm>
#include <vector>
int EditDistance(const StringPiece& s1,
diff --git a/src/graph.cc b/src/graph.cc
index 9801a7b..aa9c0e8 100644
--- a/src/graph.cc
+++ b/src/graph.cc
@@ -215,7 +215,10 @@
/// An Env for an Edge, providing $in and $out.
struct EdgeEnv : public Env {
- explicit EdgeEnv(Edge* edge) : edge_(edge) {}
+ enum EscapeKind { kShellEscape, kDoNotEscape };
+
+ explicit EdgeEnv(Edge* edge, EscapeKind escape)
+ : edge_(edge), escape_in_out_(escape) {}
virtual string LookupVariable(const string& var);
/// Given a span of Nodes, construct a list of paths suitable for a command
@@ -225,6 +228,7 @@
char sep);
Edge* edge_;
+ EscapeKind escape_in_out_;
};
string EdgeEnv::LookupVariable(const string& var) {
@@ -253,10 +257,12 @@
if (!result.empty())
result.push_back(sep);
const string& path = (*i)->path();
- if (path.find(" ") != string::npos) {
- result.append("\"");
- result.append(path);
- result.append("\"");
+ if (escape_in_out_ == kShellEscape) {
+#if _WIN32
+ GetWin32EscapedString(path, &result);
+#else
+ GetShellEscapedString(path, &result);
+#endif
} else {
result.append(path);
}
@@ -275,7 +281,7 @@
}
string Edge::GetBinding(const string& key) {
- EdgeEnv env(this);
+ EdgeEnv env(this, EdgeEnv::kShellEscape);
return env.LookupVariable(key);
}
@@ -283,6 +289,16 @@
return !GetBinding(key).empty();
}
+string Edge::GetUnescapedDepfile() {
+ EdgeEnv env(this, EdgeEnv::kDoNotEscape);
+ return env.LookupVariable("depfile");
+}
+
+string Edge::GetUnescapedRspfile() {
+ EdgeEnv env(this, EdgeEnv::kDoNotEscape);
+ return env.LookupVariable("rspfile");
+}
+
void Edge::Dump(const char* prefix) const {
printf("%s[ ", prefix);
for (vector<Node*>::const_iterator i = inputs_.begin();
@@ -308,6 +324,10 @@
return rule_ == &State::kPhonyRule;
}
+bool Edge::use_console() const {
+ return pool() == &State::kConsolePool;
+}
+
void Node::Dump(const char* prefix) const {
printf("%s <%s 0x%p> mtime: %d%s, (:%s), ",
prefix, path().c_str(), this,
@@ -330,7 +350,7 @@
if (!deps_type.empty())
return LoadDepsFromLog(edge, err);
- string depfile = edge->GetBinding("depfile");
+ string depfile = edge->GetUnescapedDepfile();
if (!depfile.empty())
return LoadDepFile(edge, depfile, err);
diff --git a/src/graph.h b/src/graph.h
index 868413c..66e31b5 100644
--- a/src/graph.h
+++ b/src/graph.h
@@ -146,9 +146,15 @@
/// full contents of a response file (if applicable)
string EvaluateCommand(bool incl_rsp_file = false);
+ /// Returns the shell-escaped value of |key|.
string GetBinding(const string& key);
bool GetBindingBool(const string& key);
+ /// Like GetBinding("depfile"), but without shell escaping.
+ string GetUnescapedDepfile();
+ /// Like GetBinding("rspfile"), but without shell escaping.
+ string GetUnescapedRspfile();
+
void Dump(const char* prefix="") const;
const Rule* rule_;
@@ -183,6 +189,7 @@
}
bool is_phony() const;
+ bool use_console() const;
};
diff --git a/src/graph_test.cc b/src/graph_test.cc
index 8521216..14dc678 100644
--- a/src/graph_test.cc
+++ b/src/graph_test.cc
@@ -139,13 +139,18 @@
}
}
-TEST_F(GraphTest, VarInOutQuoteSpaces) {
+TEST_F(GraphTest, VarInOutPathEscaping) {
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
-"build a$ b: cat nospace with$ space nospace2\n"));
+"build a$ b: cat no'space with$ space$$ no\"space2\n"));
Edge* edge = GetNode("a b")->in_edge();
- EXPECT_EQ("cat nospace \"with space\" nospace2 > \"a b\"",
+#if _WIN32
+ EXPECT_EQ("cat no'space \"with space$\" \"no\\\"space2\" > \"a b\"",
edge->EvaluateCommand());
+#else
+ EXPECT_EQ("cat 'no'\\''space' 'with space$' 'no\"space2' > 'a b'",
+ edge->EvaluateCommand());
+#endif
}
// Regression test for https://github.com/martine/ninja/issues/380
diff --git a/src/hash_map.h b/src/hash_map.h
index c63aa88..77e7586 100644
--- a/src/hash_map.h
+++ b/src/hash_map.h
@@ -15,6 +15,7 @@
#ifndef NINJA_MAP_H_
#define NINJA_MAP_H_
+#include <algorithm>
#include <string.h>
#include "string_piece.h"
diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc
index 1713d5d..419996f 100644
--- a/src/includes_normalize_test.cc
+++ b/src/includes_normalize_test.cc
@@ -38,7 +38,7 @@
} // namespace
TEST(IncludesNormalize, WithRelative) {
- string currentdir = IncludesNormalize::ToLower(GetCurDir());
+ string currentdir = GetCurDir();
EXPECT_EQ("c", IncludesNormalize::Normalize("a/b/c", "a/b"));
EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"),
NULL));
diff --git a/src/line_printer.cc b/src/line_printer.cc
index 3537e88..ef1609c 100644
--- a/src/line_printer.cc
+++ b/src/line_printer.cc
@@ -26,7 +26,7 @@
#include "util.h"
-LinePrinter::LinePrinter() : have_blank_line_(true) {
+LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) {
#ifndef _WIN32
const char* term = getenv("TERM");
smart_terminal_ = isatty(1) && term && string(term) != "dumb";
@@ -43,6 +43,12 @@
}
void LinePrinter::Print(string to_print, LineType type) {
+ if (console_locked_) {
+ line_buffer_ = to_print;
+ line_type_ = type;
+ return;
+ }
+
#ifdef _WIN32
CONSOLE_SCREEN_BUFFER_INFO csbi;
GetConsoleScreenBufferInfo(console_, &csbi);
@@ -101,13 +107,46 @@
}
}
-void LinePrinter::PrintOnNewLine(const string& to_print) {
- if (!have_blank_line_)
- printf("\n");
- if (!to_print.empty()) {
+void LinePrinter::PrintOrBuffer(const char* data, size_t size) {
+ if (console_locked_) {
+ output_buffer_.append(data, size);
+ } else {
// Avoid printf and C strings, since the actual output might contain null
// bytes like UTF-16 does (yuck).
- fwrite(&to_print[0], sizeof(char), to_print.size(), stdout);
+ fwrite(data, 1, size, stdout);
+ }
+}
+
+void LinePrinter::PrintOnNewLine(const string& to_print) {
+ if (console_locked_ && !line_buffer_.empty()) {
+ output_buffer_.append(line_buffer_);
+ output_buffer_.append(1, '\n');
+ line_buffer_.clear();
+ }
+ if (!have_blank_line_) {
+ PrintOrBuffer("\n", 1);
+ }
+ if (!to_print.empty()) {
+ PrintOrBuffer(&to_print[0], to_print.size());
}
have_blank_line_ = to_print.empty() || *to_print.rbegin() == '\n';
}
+
+void LinePrinter::SetConsoleLocked(bool locked) {
+ if (locked == console_locked_)
+ return;
+
+ if (locked)
+ PrintOnNewLine("");
+
+ console_locked_ = locked;
+
+ if (!locked) {
+ PrintOnNewLine(output_buffer_);
+ if (!line_buffer_.empty()) {
+ Print(line_buffer_, line_type_);
+ }
+ output_buffer_.clear();
+ line_buffer_.clear();
+ }
+}
diff --git a/src/line_printer.h b/src/line_printer.h
index aea2817..55225e5 100644
--- a/src/line_printer.h
+++ b/src/line_printer.h
@@ -15,6 +15,7 @@
#ifndef NINJA_LINE_PRINTER_H_
#define NINJA_LINE_PRINTER_H_
+#include <stddef.h>
#include <string>
using namespace std;
@@ -37,6 +38,10 @@
/// Prints a string on a new line, not overprinting previous output.
void PrintOnNewLine(const string& to_print);
+ /// Lock or unlock the console. Any output sent to the LinePrinter while the
+ /// console is locked will not be printed until it is unlocked.
+ void SetConsoleLocked(bool locked);
+
private:
/// Whether we can do fancy terminal control codes.
bool smart_terminal_;
@@ -44,9 +49,24 @@
/// Whether the caret is at the beginning of a blank line.
bool have_blank_line_;
+ /// Whether console is locked.
+ bool console_locked_;
+
+ /// Buffered current line while console is locked.
+ string line_buffer_;
+
+ /// Buffered line type while console is locked.
+ LineType line_type_;
+
+ /// Buffered console output while console is locked.
+ string output_buffer_;
+
#ifdef _WIN32
void* console_;
#endif
+
+ /// Print the given data to the console, or buffer it if it is locked.
+ void PrintOrBuffer(const char *data, size_t size);
};
#endif // NINJA_LINE_PRINTER_H_
diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc
index 20be7f3..6fa4f7c 100644
--- a/src/manifest_parser.cc
+++ b/src/manifest_parser.cc
@@ -296,16 +296,17 @@
if (!ExpectToken(Lexer::NEWLINE, err))
return false;
- // XXX scoped_ptr to handle error case.
- BindingEnv* env = new BindingEnv(env_);
-
- while (lexer_.PeekToken(Lexer::INDENT)) {
+ // Bindings on edges are rare, so allocate per-edge envs only when needed.
+ bool hasIdent = lexer_.PeekToken(Lexer::INDENT);
+ BindingEnv* env = hasIdent ? new BindingEnv(env_) : env_;
+ while (hasIdent) {
string key;
EvalString val;
if (!ParseLet(&key, &val, err))
return false;
env->AddBinding(key, val.Evaluate(env_));
+ hasIdent = lexer_.PeekToken(Lexer::INDENT);
}
Edge* edge = state_->AddEdge(rule);
diff --git a/src/manifest_parser_perftest.cc b/src/manifest_parser_perftest.cc
new file mode 100644
index 0000000..ca62fb2
--- /dev/null
+++ b/src/manifest_parser_perftest.cc
@@ -0,0 +1,118 @@
+// Copyright 2014 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Tests manifest parser performance. Expects to be run in ninja's root
+// directory.
+
+#include <numeric>
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#ifdef _WIN32
+#include "getopt.h"
+#include <direct.h>
+#else
+#include <getopt.h>
+#include <unistd.h>
+#endif
+
+#include "disk_interface.h"
+#include "graph.h"
+#include "manifest_parser.h"
+#include "metrics.h"
+#include "state.h"
+#include "util.h"
+
+struct RealFileReader : public ManifestParser::FileReader {
+ virtual bool ReadFile(const string& path, string* content, string* err) {
+ return ::ReadFile(path, content, err) == 0;
+ }
+};
+
+bool WriteFakeManifests(const string& dir) {
+ RealDiskInterface disk_interface;
+ if (disk_interface.Stat(dir + "/build.ninja") > 0)
+ return true;
+
+ printf("Creating manifest data..."); fflush(stdout);
+ int err = system(("python misc/write_fake_manifests.py " + dir).c_str());
+ printf("done.\n");
+ return err == 0;
+}
+
+int LoadManifests(bool measure_command_evaluation) {
+ string err;
+ RealFileReader file_reader;
+ State state;
+ ManifestParser parser(&state, &file_reader);
+ if (!parser.Load("build.ninja", &err)) {
+ fprintf(stderr, "Failed to read test data: %s\n", err.c_str());
+ exit(1);
+ }
+ // Doing an empty build involves reading the manifest and evaluating all
+ // commands required for the requested targets. So include command
+ // evaluation in the perftest by default.
+ int optimization_guard = 0;
+ if (measure_command_evaluation)
+ for (size_t i = 0; i < state.edges_.size(); ++i)
+ optimization_guard += state.edges_[i]->EvaluateCommand().size();
+ return optimization_guard;
+}
+
+int main(int argc, char* argv[]) {
+ bool measure_command_evaluation = true;
+ int opt;
+ while ((opt = getopt(argc, argv, const_cast<char*>("fh"))) != -1) {
+ switch (opt) {
+ case 'f':
+ measure_command_evaluation = false;
+ break;
+ case 'h':
+ default:
+ printf("usage: manifest_parser_perftest\n"
+"\n"
+"options:\n"
+" -f only measure manifest load time, not command evaluation time\n"
+ );
+ return 1;
+ }
+ }
+
+ const char kManifestDir[] = "build/manifest_perftest";
+
+ if (!WriteFakeManifests(kManifestDir)) {
+ fprintf(stderr, "Failed to write test data\n");
+ return 1;
+ }
+
+ if (chdir(kManifestDir) < 0)
+ Fatal("chdir: %s", strerror(errno));
+
+ const int kNumRepetitions = 5;
+ vector<int> times;
+ for (int i = 0; i < kNumRepetitions; ++i) {
+ int64_t start = GetTimeMillis();
+ int optimization_guard = LoadManifests(measure_command_evaluation);
+ int delta = (int)(GetTimeMillis() - start);
+ printf("%dms (hash: %x)\n", delta, optimization_guard);
+ times.push_back(delta);
+ }
+
+ int min = *min_element(times.begin(), times.end());
+ int max = *max_element(times.begin(), times.end());
+ float total = accumulate(times.begin(), times.end(), 0.0f);
+ printf("min %dms max %dms avg %.1fms\n", min, max, total / times.size());
+}
diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc
index 5ed1584..152b965 100644
--- a/src/manifest_parser_test.cc
+++ b/src/manifest_parser_test.cc
@@ -236,7 +236,11 @@
"build $x: foo y\n"
));
EXPECT_EQ("$dollar", state.bindings_.LookupVariable("x"));
+#ifdef _WIN32
EXPECT_EQ("$dollarbar$baz$blah", state.edges_[0]->EvaluateCommand());
+#else
+ EXPECT_EQ("'$dollar'bar$baz$blah", state.edges_[0]->EvaluateCommand());
+#endif
}
TEST_F(ParserTest, EscapeSpaces) {
@@ -762,6 +766,21 @@
, err);
}
+TEST_F(ParserTest, DuplicateRuleInDifferentSubninjas) {
+ // Test that rules live in a global namespace and aren't scoped to subninjas.
+ files_["test.ninja"] = "rule cat\n"
+ " command = cat\n";
+ ManifestParser parser(&state, this);
+ string err;
+ EXPECT_FALSE(parser.ParseTest("rule cat\n"
+ " command = cat\n"
+ "subninja test.ninja\n", &err));
+ EXPECT_EQ("test.ninja:1: duplicate rule 'cat'\n"
+ "rule cat\n"
+ " ^ near here"
+ , err);
+}
+
TEST_F(ParserTest, Include) {
files_["include.ninja"] = "var = inner\n";
ASSERT_NO_FATAL_FAILURE(AssertParse(
diff --git a/src/metrics.cc b/src/metrics.cc
index ca4f97a..a7d3c7a 100644
--- a/src/metrics.cc
+++ b/src/metrics.cc
@@ -24,6 +24,8 @@
#include <windows.h>
#endif
+#include <algorithm>
+
#include "util.h"
Metrics* g_metrics = NULL;
diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc
index 7c45029..e465279 100644
--- a/src/msvc_helper-win32.cc
+++ b/src/msvc_helper-win32.cc
@@ -48,14 +48,15 @@
}
// static
-string CLParser::FilterShowIncludes(const string& line) {
- static const char kMagicPrefix[] = "Note: including file: ";
+string CLParser::FilterShowIncludes(const string& line,
+ const string& deps_prefix) {
+ const string kDepsPrefixEnglish = "Note: including file: ";
const char* in = line.c_str();
const char* end = in + line.size();
-
- if (end - in > (int)sizeof(kMagicPrefix) - 1 &&
- memcmp(in, kMagicPrefix, sizeof(kMagicPrefix) - 1) == 0) {
- in += sizeof(kMagicPrefix) - 1;
+ const string& prefix = deps_prefix.empty() ? kDepsPrefixEnglish : deps_prefix;
+ if (end - in > (int)prefix.size() &&
+ memcmp(in, prefix.c_str(), (int)prefix.size()) == 0) {
+ in += prefix.size();
while (*in == ' ')
++in;
return line.substr(in - line.c_str());
@@ -81,7 +82,7 @@
EndsWith(line, ".cpp");
}
-string CLParser::Parse(const string& output) {
+string CLParser::Parse(const string& output, const string& deps_prefix) {
string filtered_output;
// Loop over all lines in the output to process them.
@@ -92,7 +93,7 @@
end = output.size();
string line = output.substr(start, end - start);
- string include = FilterShowIncludes(line);
+ string include = FilterShowIncludes(line, deps_prefix);
if (!include.empty()) {
include = IncludesNormalize::Normalize(include, NULL);
if (!IsSystemInclude(include))
@@ -140,7 +141,7 @@
STARTUPINFO startup_info = {};
startup_info.cb = sizeof(STARTUPINFO);
startup_info.hStdInput = nul;
- startup_info.hStdError = stdout_write;
+ startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE);
startup_info.hStdOutput = stdout_write;
startup_info.dwFlags |= STARTF_USESTDHANDLES;
diff --git a/src/msvc_helper.h b/src/msvc_helper.h
index e207485..5d7dcb0 100644
--- a/src/msvc_helper.h
+++ b/src/msvc_helper.h
@@ -27,7 +27,8 @@
/// Parse a line of cl.exe output and extract /showIncludes info.
/// If a dependency is extracted, returns a nonempty string.
/// Exposed for testing.
- static string FilterShowIncludes(const string& line);
+ static string FilterShowIncludes(const string& line,
+ const string& deps_prefix);
/// Return true if a mentioned include file is a system path.
/// Filtering these out reduces dependency information considerably.
@@ -41,7 +42,7 @@
/// Parse the full output of cl, returning the output (if any) that
/// should printed.
- string Parse(const string& output);
+ string Parse(const string& output, const string& deps_prefix);
set<string> includes_;
};
diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc
index e3a7846..58bc797 100644
--- a/src/msvc_helper_main-win32.cc
+++ b/src/msvc_helper_main-win32.cc
@@ -31,6 +31,7 @@
"options:\n"
" -e ENVFILE load environment block from ENVFILE as environment\n"
" -o FILE write output dependency information to FILE.d\n"
+" -p STRING localized prefix of msvc's /showIncludes output\n"
);
}
@@ -84,7 +85,8 @@
{ NULL, 0, NULL, 0 }
};
int opt;
- while ((opt = getopt_long(argc, argv, "e:o:h", kLongOptions, NULL)) != -1) {
+ string deps_prefix;
+ while ((opt = getopt_long(argc, argv, "e:o:p:h", kLongOptions, NULL)) != -1) {
switch (opt) {
case 'e':
envfile = optarg;
@@ -92,6 +94,9 @@
case 'o':
output_filename = optarg;
break;
+ case 'p':
+ deps_prefix = optarg;
+ break;
case 'h':
default:
Usage();
@@ -122,7 +127,7 @@
if (output_filename) {
CLParser parser;
- output = parser.Parse(output);
+ output = parser.Parse(output, deps_prefix);
WriteDepFileOrDie(output_filename, parser);
}
diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc
index 02f2863..391c045 100644
--- a/src/msvc_helper_test.cc
+++ b/src/msvc_helper_test.cc
@@ -20,15 +20,19 @@
#include "util.h"
TEST(CLParserTest, ShowIncludes) {
- ASSERT_EQ("", CLParser::FilterShowIncludes(""));
+ ASSERT_EQ("", CLParser::FilterShowIncludes("", ""));
- ASSERT_EQ("", CLParser::FilterShowIncludes("Sample compiler output"));
+ ASSERT_EQ("", CLParser::FilterShowIncludes("Sample compiler output", ""));
ASSERT_EQ("c:\\Some Files\\foobar.h",
CLParser::FilterShowIncludes("Note: including file: "
- "c:\\Some Files\\foobar.h"));
+ "c:\\Some Files\\foobar.h", ""));
ASSERT_EQ("c:\\initspaces.h",
CLParser::FilterShowIncludes("Note: including file: "
- "c:\\initspaces.h"));
+ "c:\\initspaces.h", ""));
+ ASSERT_EQ("c:\\initspaces.h",
+ CLParser::FilterShowIncludes("Non-default prefix: inc file: "
+ "c:\\initspaces.h",
+ "Non-default prefix: inc file:"));
}
TEST(CLParserTest, FilterInputFilename) {
@@ -46,8 +50,9 @@
CLParser parser;
string output = parser.Parse(
"foo\r\n"
- "Note: including file: foo.h\r\n"
- "bar\r\n");
+ "Note: inc file prefix: foo.h\r\n"
+ "bar\r\n",
+ "Note: inc file prefix:");
ASSERT_EQ("foo\nbar\n", output);
ASSERT_EQ(1u, parser.includes_.size());
@@ -58,7 +63,8 @@
CLParser parser;
string output = parser.Parse(
"foo.cc\r\n"
- "cl: warning\r\n");
+ "cl: warning\r\n",
+ "");
ASSERT_EQ("cl: warning\n", output);
}
@@ -67,7 +73,8 @@
string output = parser.Parse(
"Note: including file: c:\\Program Files\\foo.h\r\n"
"Note: including file: d:\\Microsoft Visual Studio\\bar.h\r\n"
- "Note: including file: path.h\r\n");
+ "Note: including file: path.h\r\n",
+ "");
// We should have dropped the first two includes because they look like
// system headers.
ASSERT_EQ("", output);
@@ -80,7 +87,8 @@
string output = parser.Parse(
"Note: including file: foo.h\r\n"
"Note: including file: bar.h\r\n"
- "Note: including file: foo.h\r\n");
+ "Note: including file: foo.h\r\n",
+ "");
// We should have dropped one copy of foo.h.
ASSERT_EQ("", output);
ASSERT_EQ(2u, parser.includes_.size());
@@ -91,7 +99,8 @@
string output = parser.Parse(
"Note: including file: sub/foo.h\r\n"
"Note: including file: bar.h\r\n"
- "Note: including file: sub\\foo.h\r\n");
+ "Note: including file: sub\\foo.h\r\n",
+ "");
// We should have dropped one copy of foo.h.
ASSERT_EQ("", output);
ASSERT_EQ(2u, parser.includes_.size());
@@ -110,3 +119,10 @@
cl.Run("cmd /c \"echo foo is %foo%", &output);
ASSERT_EQ("foo is bar\r\n", output);
}
+
+TEST(MSVCHelperTest, NoReadOfStderr) {
+ CLWrapper cl;
+ string output;
+ cl.Run("cmd /c \"echo to stdout&& echo to stderr 1>&2", &output);
+ ASSERT_EQ("to stdout\r\n", output);
+}
diff --git a/src/ninja.cc b/src/ninja.cc
index a313ecb..a381e83 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -68,7 +68,7 @@
/// The Ninja main() loads up a series of data structures; various tools need
/// to poke into these, so store them as fields on an object.
-struct NinjaMain {
+struct NinjaMain : public BuildLogUser {
NinjaMain(const char* ninja_command, const BuildConfig& config) :
ninja_command_(ninja_command), config_(config) {}
@@ -137,6 +137,20 @@
/// Dump the output requested by '-d stats'.
void DumpMetrics();
+
+ virtual bool IsPathDead(StringPiece s) const {
+ Node* n = state_.LookupNode(s);
+ // Just checking n isn't enough: If an old output is both in the build log
+ // and in the deps log, it will have a Node object in state_. (It will also
+ // have an in edge if one of its inputs is another output that's in the deps
+ // log, but having a deps edge product an output thats input to another deps
+ // edge is rare, and the first recompaction will delete all old outputs from
+ // the deps log, and then a second recompaction will clear the build log,
+ // which seems good enough for this corner case.)
+ // Do keep entries around for files which still exist on disk, for
+ // generators that want to use this information.
+ return (!n || !n->in_edge()) && disk_interface_.Stat(s.AsString()) == 0;
+ }
};
/// Subtools, accessible via "-t foo".
@@ -444,9 +458,7 @@
if (argc == 0) {
for (vector<Node*>::const_iterator ni = deps_log_.nodes().begin();
ni != deps_log_.nodes().end(); ++ni) {
- // Only query for targets with an incoming edge and deps
- Edge* e = (*ni)->in_edge();
- if (e && !e->GetBinding("deps").empty())
+ if (deps_log_.IsDepsEntryLiveFor(*ni))
nodes.push_back(*ni);
}
} else {
@@ -621,6 +633,8 @@
putchar('[');
for (vector<Edge*>::iterator e = state_.edges_.begin();
e != state_.edges_.end(); ++e) {
+ if ((*e)->inputs_.empty())
+ continue;
for (int i = 0; i != argc; ++i) {
if ((*e)->rule_->name() == argv[i]) {
if (!first)
@@ -748,6 +762,9 @@
" stats print operation counts/timing info\n"
" explain explain what caused a command to execute\n"
" keeprsp don't delete @response files on success\n"
+#ifdef _WIN32
+" nostatcache don't batch stat() calls per directory and cache them\n"
+#endif
"multiple modes can be enabled via -d FOO -d BAR\n");
return false;
} else if (name == "stats") {
@@ -759,9 +776,13 @@
} else if (name == "keeprsp") {
g_keep_rsp = true;
return true;
+ } else if (name == "nostatcache") {
+ g_experimental_statcache = false;
+ return true;
} else {
const char* suggestion =
- SpellcheckString(name.c_str(), "stats", "explain", NULL);
+ SpellcheckString(name.c_str(), "stats", "explain", "keeprsp",
+ "nostatcache", NULL);
if (suggestion) {
Error("unknown debug setting '%s', did you mean '%s'?",
name.c_str(), suggestion);
@@ -789,14 +810,14 @@
}
if (recompact_only) {
- bool success = build_log_.Recompact(log_path, &err);
+ bool success = build_log_.Recompact(log_path, *this, &err);
if (!success)
Error("failed recompaction: %s", err.c_str());
return success;
}
if (!config_.dry_run) {
- if (!build_log_.OpenForWrite(log_path, &err)) {
+ if (!build_log_.OpenForWrite(log_path, *this, &err)) {
Error("opening build log: %s", err.c_str());
return false;
}
@@ -870,6 +891,8 @@
return 1;
}
+ disk_interface_.AllowStatCache(g_experimental_statcache);
+
Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
for (size_t i = 0; i < targets.size(); ++i) {
if (!builder.AddTarget(targets[i], &err)) {
@@ -883,6 +906,9 @@
}
}
+ // Make sure restat rules do not see stale timestamps.
+ disk_interface_.AllowStatCache(false);
+
if (builder.AlreadyUpToDate()) {
printf("ninja: no work to do.\n");
return 0;
diff --git a/src/state.cc b/src/state.cc
index 9b6160b..7258272 100644
--- a/src/state.cc
+++ b/src/state.cc
@@ -69,11 +69,13 @@
}
Pool State::kDefaultPool("", 0);
+Pool State::kConsolePool("console", 1);
const Rule State::kPhonyRule("phony");
State::State() {
AddRule(&kPhonyRule);
AddPool(&kDefaultPool);
+ AddPool(&kConsolePool);
}
void State::AddRule(const Rule* rule) {
@@ -118,9 +120,9 @@
return node;
}
-Node* State::LookupNode(StringPiece path) {
+Node* State::LookupNode(StringPiece path) const {
METRIC_RECORD("lookup node");
- Paths::iterator i = paths_.find(path);
+ Paths::const_iterator i = paths_.find(path);
if (i != paths_.end())
return i->second;
return NULL;
diff --git a/src/state.h b/src/state.h
index bde75ff..c382dc0 100644
--- a/src/state.h
+++ b/src/state.h
@@ -82,6 +82,7 @@
/// Global state (file status, loaded rules) for a single run.
struct State {
static Pool kDefaultPool;
+ static Pool kConsolePool;
static const Rule kPhonyRule;
State();
@@ -95,7 +96,7 @@
Edge* AddEdge(const Rule* rule);
Node* GetNode(StringPiece path);
- Node* LookupNode(StringPiece path);
+ Node* LookupNode(StringPiece path) const;
Node* SpellcheckNode(const string& path);
void AddIn(Edge* edge, StringPiece path);
diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc
index a9af756..743e406 100644
--- a/src/subprocess-posix.cc
+++ b/src/subprocess-posix.cc
@@ -25,7 +25,8 @@
#include "util.h"
-Subprocess::Subprocess() : fd_(-1), pid_(-1) {
+Subprocess::Subprocess(bool use_console) : fd_(-1), pid_(-1),
+ use_console_(use_console) {
}
Subprocess::~Subprocess() {
if (fd_ >= 0)
@@ -58,29 +59,34 @@
// Track which fd we use to report errors on.
int error_pipe = output_pipe[1];
do {
- if (setpgid(0, 0) < 0)
- break;
-
if (sigaction(SIGINT, &set->old_act_, 0) < 0)
break;
if (sigprocmask(SIG_SETMASK, &set->old_mask_, 0) < 0)
break;
- // Open /dev/null over stdin.
- int devnull = open("/dev/null", O_RDONLY);
- if (devnull < 0)
- break;
- if (dup2(devnull, 0) < 0)
- break;
- close(devnull);
+ if (!use_console_) {
+ // Put the child in its own process group, so ctrl-c won't reach it.
+ if (setpgid(0, 0) < 0)
+ break;
- if (dup2(output_pipe[1], 1) < 0 ||
- dup2(output_pipe[1], 2) < 0)
- break;
+ // Open /dev/null over stdin.
+ int devnull = open("/dev/null", O_RDONLY);
+ if (devnull < 0)
+ break;
+ if (dup2(devnull, 0) < 0)
+ break;
+ close(devnull);
- // Now can use stderr for errors.
- error_pipe = 2;
- close(output_pipe[1]);
+ if (dup2(output_pipe[1], 1) < 0 ||
+ dup2(output_pipe[1], 2) < 0)
+ break;
+
+ // Now can use stderr for errors.
+ error_pipe = 2;
+ close(output_pipe[1]);
+ }
+ // In the console case, output_pipe is still inherited by the child and
+ // closed when the subprocess finishes, which then notifies ninja.
execl("/bin/sh", "/bin/sh", "-c", command.c_str(), (char *) NULL);
} while (false);
@@ -168,8 +174,8 @@
Fatal("sigprocmask: %s", strerror(errno));
}
-Subprocess *SubprocessSet::Add(const string& command) {
- Subprocess *subprocess = new Subprocess;
+Subprocess *SubprocessSet::Add(const string& command, bool use_console) {
+ Subprocess *subprocess = new Subprocess(use_console);
if (!subprocess->Start(this, command)) {
delete subprocess;
return 0;
@@ -279,7 +285,10 @@
void SubprocessSet::Clear() {
for (vector<Subprocess*>::iterator i = running_.begin();
i != running_.end(); ++i)
- kill(-(*i)->pid_, SIGINT);
+ // Since the foreground process is in our process group, it will receive a
+ // SIGINT at the same time as us.
+ if (!(*i)->use_console_)
+ kill(-(*i)->pid_, SIGINT);
for (vector<Subprocess*>::iterator i = running_.begin();
i != running_.end(); ++i)
delete *i;
diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc
index 1b230b6..fad66e8 100644
--- a/src/subprocess-win32.cc
+++ b/src/subprocess-win32.cc
@@ -14,13 +14,16 @@
#include "subprocess.h"
+#include <assert.h>
#include <stdio.h>
#include <algorithm>
#include "util.h"
-Subprocess::Subprocess() : child_(NULL) , overlapped_(), is_reading_(false) {
+Subprocess::Subprocess(bool use_console) : child_(NULL) , overlapped_(),
+ is_reading_(false),
+ use_console_(use_console) {
}
Subprocess::~Subprocess() {
@@ -86,18 +89,25 @@
STARTUPINFOA startup_info;
memset(&startup_info, 0, sizeof(startup_info));
startup_info.cb = sizeof(STARTUPINFO);
- startup_info.dwFlags = STARTF_USESTDHANDLES;
- startup_info.hStdInput = nul;
- startup_info.hStdOutput = child_pipe;
- startup_info.hStdError = child_pipe;
+ if (!use_console_) {
+ startup_info.dwFlags = STARTF_USESTDHANDLES;
+ startup_info.hStdInput = nul;
+ startup_info.hStdOutput = child_pipe;
+ startup_info.hStdError = child_pipe;
+ }
+ // In the console case, child_pipe is still inherited by the child and closed
+ // when the subprocess finishes, which then notifies ninja.
PROCESS_INFORMATION process_info;
memset(&process_info, 0, sizeof(process_info));
+ // Ninja handles ctrl-c, except for subprocesses in console pools.
+ DWORD process_flags = use_console_ ? 0 : CREATE_NEW_PROCESS_GROUP;
+
// Do not prepend 'cmd /c' on Windows, this breaks command
// lines greater than 8,191 chars.
if (!CreateProcessA(NULL, (char*)command.c_str(), NULL, NULL,
- /* inherit handles */ TRUE, CREATE_NEW_PROCESS_GROUP,
+ /* inherit handles */ TRUE, process_flags,
NULL, NULL,
&startup_info, &process_info)) {
DWORD error = GetLastError();
@@ -213,8 +223,8 @@
return FALSE;
}
-Subprocess *SubprocessSet::Add(const string& command) {
- Subprocess *subprocess = new Subprocess;
+Subprocess *SubprocessSet::Add(const string& command, bool use_console) {
+ Subprocess *subprocess = new Subprocess(use_console);
if (!subprocess->Start(this, command)) {
delete subprocess;
return 0;
@@ -266,7 +276,9 @@
void SubprocessSet::Clear() {
for (vector<Subprocess*>::iterator i = running_.begin();
i != running_.end(); ++i) {
- if ((*i)->child_) {
+ // Since the foreground process is in our process group, it will receive a
+ // CTRL_C_EVENT or CTRL_BREAK_EVENT at the same time as us.
+ if ((*i)->child_ && !(*i)->use_console_) {
if (!GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT,
GetProcessId((*i)->child_))) {
Win32Fatal("GenerateConsoleCtrlEvent");
diff --git a/src/subprocess.h b/src/subprocess.h
index 4c1629c..b7a1a4c 100644
--- a/src/subprocess.h
+++ b/src/subprocess.h
@@ -44,7 +44,7 @@
const string& GetOutput() const;
private:
- Subprocess();
+ Subprocess(bool use_console);
bool Start(struct SubprocessSet* set, const string& command);
void OnPipeReady();
@@ -64,6 +64,7 @@
int fd_;
pid_t pid_;
#endif
+ bool use_console_;
friend struct SubprocessSet;
};
@@ -75,7 +76,7 @@
SubprocessSet();
~SubprocessSet();
- Subprocess* Add(const string& command);
+ Subprocess* Add(const string& command, bool use_console = false);
bool DoWork();
Subprocess* NextFinished();
void Clear();
diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc
index 9f8dcea..775a13a 100644
--- a/src/subprocess_test.cc
+++ b/src/subprocess_test.cc
@@ -95,6 +95,21 @@
ADD_FAILURE() << "We should have been interrupted";
}
+TEST_F(SubprocessTest, Console) {
+ // Skip test if we don't have the console ourselves.
+ if (isatty(0) && isatty(1) && isatty(2)) {
+ Subprocess* subproc = subprocs_.Add("test -t 0 -a -t 1 -a -t 2",
+ /*use_console=*/true);
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ while (!subproc->Done()) {
+ subprocs_.DoWork();
+ }
+
+ EXPECT_EQ(ExitSuccess, subproc->Finish());
+ }
+}
+
#endif
TEST_F(SubprocessTest, SetWithSingle) {
diff --git a/src/test.cc b/src/test.cc
index 45a9226..21015ed 100644
--- a/src/test.cc
+++ b/src/test.cc
@@ -105,8 +105,8 @@
files_created_.insert(path);
}
-TimeStamp VirtualFileSystem::Stat(const string& path) {
- FileMap::iterator i = files_.find(path);
+TimeStamp VirtualFileSystem::Stat(const string& path) const {
+ FileMap::const_iterator i = files_.find(path);
if (i != files_.end())
return i->second.mtime;
return 0;
diff --git a/src/test.h b/src/test.h
index 9f29e07..f34b877 100644
--- a/src/test.h
+++ b/src/test.h
@@ -59,7 +59,7 @@
}
// DiskInterface
- virtual TimeStamp Stat(const string& path);
+ virtual TimeStamp Stat(const string& path) const;
virtual bool WriteFile(const string& path, const string& contents);
virtual bool MakeDir(const string& path);
virtual string ReadFile(const string& path, string* err);
diff --git a/src/util.cc b/src/util.cc
index 6ba3c6c..484b0c1 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -20,6 +20,7 @@
#include <share.h>
#endif
+#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <stdarg.h>
@@ -175,6 +176,109 @@
return true;
}
+static inline bool IsKnownShellSafeCharacter(char ch) {
+ if ('A' <= ch && ch <= 'Z') return true;
+ if ('a' <= ch && ch <= 'z') return true;
+ if ('0' <= ch && ch <= '9') return true;
+
+ switch (ch) {
+ case '_':
+ case '+':
+ case '-':
+ case '.':
+ case '/':
+ return true;
+ default:
+ return false;
+ }
+}
+
+static inline bool IsKnownWin32SafeCharacter(char ch) {
+ switch (ch) {
+ case ' ':
+ case '"':
+ return false;
+ default:
+ return true;
+ }
+}
+
+static inline bool StringNeedsShellEscaping(const string& input) {
+ for (size_t i = 0; i < input.size(); ++i) {
+ if (!IsKnownShellSafeCharacter(input[i])) return true;
+ }
+ return false;
+}
+
+static inline bool StringNeedsWin32Escaping(const string& input) {
+ for (size_t i = 0; i < input.size(); ++i) {
+ if (!IsKnownWin32SafeCharacter(input[i])) return true;
+ }
+ return false;
+}
+
+void GetShellEscapedString(const string& input, string* result) {
+ assert(result);
+
+ if (!StringNeedsShellEscaping(input)) {
+ result->append(input);
+ return;
+ }
+
+ const char kQuote = '\'';
+ const char kEscapeSequence[] = "'\\'";
+
+ result->push_back(kQuote);
+
+ string::const_iterator span_begin = input.begin();
+ for (string::const_iterator it = input.begin(), end = input.end(); it != end;
+ ++it) {
+ if (*it == kQuote) {
+ result->append(span_begin, it);
+ result->append(kEscapeSequence);
+ span_begin = it;
+ }
+ }
+ result->append(span_begin, input.end());
+ result->push_back(kQuote);
+}
+
+
+void GetWin32EscapedString(const string& input, string* result) {
+ assert(result);
+ if (!StringNeedsWin32Escaping(input)) {
+ result->append(input);
+ return;
+ }
+
+ const char kQuote = '"';
+ const char kBackslash = '\\';
+
+ result->push_back(kQuote);
+ size_t consecutive_backslash_count = 0;
+ string::const_iterator span_begin = input.begin();
+ for (string::const_iterator it = input.begin(), end = input.end(); it != end;
+ ++it) {
+ switch (*it) {
+ case kBackslash:
+ ++consecutive_backslash_count;
+ break;
+ case kQuote:
+ result->append(span_begin, it);
+ result->append(consecutive_backslash_count + 1, kBackslash);
+ span_begin = it;
+ consecutive_backslash_count = 0;
+ break;
+ default:
+ consecutive_backslash_count = 0;
+ break;
+ }
+ }
+ result->append(span_begin, input.end());
+ result->append(consecutive_backslash_count, kBackslash);
+ result->push_back(kQuote);
+}
+
int ReadFile(const string& path, string* contents, string* err) {
FILE* f = fopen(path.c_str(), "r");
if (!f) {
diff --git a/src/util.h b/src/util.h
index 6788410..7101770 100644
--- a/src/util.h
+++ b/src/util.h
@@ -45,6 +45,13 @@
bool CanonicalizePath(char* path, size_t* len, string* err);
+/// Appends |input| to |*result|, escaping according to the whims of either
+/// Bash, or Win32's CommandLineToArgvW().
+/// Appends the string directly to |result| without modification if we can
+/// determine that it contains no problematic characters.
+void GetShellEscapedString(const string& input, string* result);
+void GetWin32EscapedString(const string& input, string* result);
+
/// Read a file to a string (in text mode: with CRLF conversion
/// on Windows).
/// Returns -errno and fills in \a err on error.
diff --git a/src/util_test.cc b/src/util_test.cc
index 1e29053..b58d15e 100644
--- a/src/util_test.cc
+++ b/src/util_test.cc
@@ -136,6 +136,37 @@
EXPECT_EQ("file ./file bar/.", string(path));
}
+TEST(PathEscaping, TortureTest) {
+ string result;
+
+ GetWin32EscapedString("foo bar\\\"'$@d!st!c'\\path'\\", &result);
+ EXPECT_EQ("\"foo bar\\\\\\\"'$@d!st!c'\\path'\\\\\"", result);
+ result.clear();
+
+ GetShellEscapedString("foo bar\"/'$@d!st!c'/path'", &result);
+ EXPECT_EQ("'foo bar\"/'\\''$@d!st!c'\\''/path'\\'''", result);
+}
+
+TEST(PathEscaping, SensiblePathsAreNotNeedlesslyEscaped) {
+ const char* path = "some/sensible/path/without/crazy/characters.c++";
+ string result;
+
+ GetWin32EscapedString(path, &result);
+ EXPECT_EQ(path, result);
+ result.clear();
+
+ GetShellEscapedString(path, &result);
+ EXPECT_EQ(path, result);
+}
+
+TEST(PathEscaping, SensibleWin32PathsAreNotNeedlesslyEscaped) {
+ const char* path = "some\\sensible\\path\\without\\crazy\\characters.c++";
+ string result;
+
+ GetWin32EscapedString(path, &result);
+ EXPECT_EQ(path, result);
+}
+
TEST(StripAnsiEscapeCodes, EscapeAtEnd) {
string stripped = StripAnsiEscapeCodes("foo\33");
EXPECT_EQ("foo", stripped);
diff --git a/src/version.cc b/src/version.cc
index 17c71aa..6d2d37c 100644
--- a/src/version.cc
+++ b/src/version.cc
@@ -18,7 +18,7 @@
#include "util.h"
-const char* kNinjaVersion = "1.4.0";
+const char* kNinjaVersion = "1.5.0";
void ParseVersion(const string& version, int* major, int* minor) {
size_t end = version.find('.');
