Merge branch 'master' into release
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..d7bee6f
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,8 @@
+language: cpp
+compiler:
+ - gcc
+ - clang
+before_install:
+ - sudo apt-get update -qq
+ - sudo apt-get install libgtest-dev
+script: ./bootstrap.py && ./configure.py --with-gtest=/usr/src/gtest && ./ninja ninja_test && ./ninja_test --gtest_filter=-SubprocessTest.SetWithLots
diff --git a/HACKING.md b/HACKING.md
index 885d2d7..11edf74 100644
--- a/HACKING.md
+++ b/HACKING.md
@@ -172,3 +172,9 @@
* Build `ninja.exe` using a Linux ninja binary: `/path/to/linux/ninja`
* Run: `./ninja.exe` (implicitly runs through wine(!))
+### Using Microsoft compilers on Linux (extremely flaky)
+
+The trick is to install just the compilers, and not all of Visual Studio,
+by following [these instructions][win7sdk].
+
+[win7sdk]: http://www.kegel.com/wine/cl-howto-win7sdk.html
diff --git a/RELEASING b/RELEASING
index 0593e6d..4d2e46a 100644
--- a/RELEASING
+++ b/RELEASING
@@ -1,9 +1,9 @@
Notes to myself on all the steps to make for a Ninja release.
-1. git checkout release; git merge master
-2. fix version number in source (it will likely conflict in the above)
-3. fix version in doc/manual.asciidoc
-4. grep doc/manual.asciidoc for XXX, fix version references
+1. update src/version.cc with new version (with ".git")
+2. git checkout release; git merge master
+3. fix version number in src/version.cc (it will likely conflict in the above)
+4. fix version in doc/manual.asciidoc
5. rebuild manual, put in place on website
6. commit, tag, push
7. construct release notes from prior notes
diff --git a/bootstrap.py b/bootstrap.py
index fcf1a20..cff10ba 100755
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -23,20 +23,25 @@
import shlex
import shutil
import subprocess
+import platform_helper
os.chdir(os.path.dirname(os.path.abspath(__file__)))
parser = OptionParser()
+
parser.add_option('--verbose', action='store_true',
help='enable verbose build',)
parser.add_option('--x64', action='store_true',
help='force 64-bit build (Windows)',)
-# TODO: make this --platform to match configure.py.
-parser.add_option('--windows', action='store_true',
- help='force native Windows build (when using Cygwin Python)',
- default=sys.platform.startswith('win32'))
+parser.add_option('--platform',
+ help='target platform (' + '/'.join(platform_helper.platforms()) + ')',
+ choices=platform_helper.platforms())
(options, conf_args) = parser.parse_args()
+
+platform = platform_helper.Platform(options.platform)
+conf_args.append("--platform=" + platform.platform())
+
def run(*args, **kwargs):
returncode = subprocess.call(*args, **kwargs)
if returncode != 0:
@@ -46,7 +51,7 @@
# g++ call as well as in the later configure.py.
cflags = os.environ.get('CFLAGS', '').split()
ldflags = os.environ.get('LDFLAGS', '').split()
-if sys.platform.startswith('freebsd'):
+if platform.is_freebsd() or platform.is_openbsd():
cflags.append('-I/usr/local/include')
ldflags.append('-L/usr/local/lib')
@@ -70,7 +75,7 @@
if filename == 'browse.cc': # Depends on generated header.
continue
- if options.windows:
+ if platform.is_windows():
if src.endswith('-posix.cc'):
continue
else:
@@ -79,35 +84,36 @@
sources.append(src)
-if options.windows:
+if platform.is_windows():
sources.append('src/getopt.c')
-vcdir = os.environ.get('VCINSTALLDIR')
-if vcdir:
- if options.x64:
- cl = [os.path.join(vcdir, 'bin', 'x86_amd64', 'cl.exe')]
- if not os.path.exists(cl[0]):
- cl = [os.path.join(vcdir, 'bin', 'amd64', 'cl.exe')]
- else:
- cl = [os.path.join(vcdir, 'bin', 'cl.exe')]
- args = cl + ['/nologo', '/EHsc', '/DNOMINMAX']
+if platform.is_msvc():
+ cl = 'cl'
+ vcdir = os.environ.get('VCINSTALLDIR')
+ if vcdir:
+ if options.x64:
+ cl = os.path.join(vcdir, 'bin', 'x86_amd64', 'cl.exe')
+ if not os.path.exists(cl):
+ cl = os.path.join(vcdir, 'bin', 'amd64', 'cl.exe')
+ else:
+ cl = os.path.join(vcdir, 'bin', 'cl.exe')
+ args = [cl, '/nologo', '/EHsc', '/DNOMINMAX']
else:
args = shlex.split(os.environ.get('CXX', 'g++'))
cflags.extend(['-Wno-deprecated',
'-DNINJA_PYTHON="' + sys.executable + '"',
'-DNINJA_BOOTSTRAP'])
- if options.windows:
+ if platform.is_windows():
cflags.append('-D_WIN32_WINNT=0x0501')
- conf_args.append("--platform=mingw")
if options.x64:
cflags.append('-m64')
args.extend(cflags)
args.extend(ldflags)
binary = 'ninja.bootstrap'
-if options.windows:
+if platform.is_windows():
binary = 'ninja.bootstrap.exe'
args.extend(sources)
-if vcdir:
+if platform.is_msvc():
args.extend(['/link', '/out:' + binary])
else:
args.extend(['-o', binary])
@@ -125,10 +131,9 @@
if options.verbose:
verbose = ['-v']
-if options.windows:
+if platform.is_windows():
print('Building ninja using itself...')
- run([sys.executable, 'configure.py', '--with-ninja=%s' % binary] +
- conf_args)
+ run([sys.executable, 'configure.py'] + conf_args)
run(['./' + binary] + verbose)
# Copy the new executable over the bootstrap one.
@@ -143,9 +148,7 @@
Note: to work around Windows file locking, where you can't rebuild an
in-use binary, to run ninja after making any changes to build ninja itself
-you should run ninja.bootstrap instead. Your build is also configured to
-use ninja.bootstrap.exe as the MSVC helper; see the --with-ninja flag of
-the --help output of configure.py.""")
+you should run ninja.bootstrap instead.""")
else:
print('Building ninja using itself...')
run([sys.executable, 'configure.py'] + conf_args)
diff --git a/configure.py b/configure.py
index 10c6994..1284deb 100755
--- a/configure.py
+++ b/configure.py
@@ -24,19 +24,19 @@
from optparse import OptionParser
import os
import sys
+import platform_helper
sys.path.insert(0, 'misc')
import ninja_syntax
parser = OptionParser()
-platforms = ['linux', 'freebsd', 'solaris', 'mingw', 'windows']
profilers = ['gmon', 'pprof']
parser.add_option('--platform',
- help='target platform (' + '/'.join(platforms) + ')',
- choices=platforms)
+ help='target platform (' + '/'.join(platform_helper.platforms()) + ')',
+ choices=platform_helper.platforms())
parser.add_option('--host',
- help='host platform (' + '/'.join(platforms) + ')',
- choices=platforms)
+ help='host platform (' + '/'.join(platform_helper.platforms()) + ')',
+ choices=platform_helper.platforms())
parser.add_option('--debug', action='store_true',
help='enable debugging extras',)
parser.add_option('--profile', metavar='TYPE',
@@ -47,28 +47,16 @@
parser.add_option('--with-python', metavar='EXE',
help='use EXE as the Python interpreter',
default=os.path.basename(sys.executable))
-parser.add_option('--with-ninja', metavar='NAME',
- help="name for ninja binary for -t msvc (MSVC only)",
- default="ninja")
(options, args) = parser.parse_args()
if args:
print('ERROR: extra unparsed command-line arguments:', args)
sys.exit(1)
-platform = options.platform
-if platform is None:
- platform = sys.platform
- if platform.startswith('linux'):
- platform = 'linux'
- elif platform.startswith('freebsd'):
- platform = 'freebsd'
- elif platform.startswith('solaris'):
- platform = 'solaris'
- elif platform.startswith('mingw'):
- platform = 'mingw'
- elif platform.startswith('win'):
- platform = 'windows'
-host = options.host or platform
+platform = platform_helper.Platform(options.platform)
+if options.host:
+ host = platform_helper.Platform(options.host)
+else:
+ host = platform
BUILD_FILENAME = 'build.ninja'
buildfile = open(BUILD_FILENAME, 'w')
@@ -88,7 +76,7 @@
CXX = configure_env.get('CXX', 'g++')
objext = '.o'
-if platform == 'windows':
+if platform.is_msvc():
CXX = 'cl'
objext = '.obj'
@@ -103,7 +91,7 @@
def cxx(name, **kwargs):
return n.build(built(name + objext), 'cxx', src(name + '.cc'), **kwargs)
def binary(name):
- if platform in ('mingw', 'windows'):
+ if platform.is_windows():
exe = name + '.exe'
n.build(name, 'phony', exe)
return exe
@@ -111,12 +99,12 @@
n.variable('builddir', 'build')
n.variable('cxx', CXX)
-if platform == 'windows':
+if platform.is_msvc():
n.variable('ar', 'link')
else:
n.variable('ar', configure_env.get('AR', 'ar'))
-if platform == 'windows':
+if platform.is_msvc():
cflags = ['/nologo', # Don't print startup banner.
'/Zi', # Create pdb with debug info.
'/W4', # Highest warning level.
@@ -130,6 +118,7 @@
# We never have strings or arrays larger than 2**31.
'/wd4267',
'/DNOMINMAX', '/D_CRT_SECURE_NO_WARNINGS',
+ '/D_VARIADIC_MAX=10',
'/DNINJA_PYTHON="%s"' % options.with_python]
ldflags = ['/DEBUG', '/libpath:$builddir']
if not options.debug:
@@ -151,31 +140,32 @@
cflags += ['-O2', '-DNDEBUG']
if 'clang' in os.path.basename(CXX):
cflags += ['-fcolor-diagnostics']
- if platform == 'mingw':
+ if platform.is_mingw():
cflags += ['-D_WIN32_WINNT=0x0501']
ldflags = ['-L$builddir']
libs = []
-if platform == 'mingw':
+if platform.is_mingw():
cflags.remove('-fvisibility=hidden');
ldflags.append('-static')
-elif platform == 'sunos5':
+elif platform.is_sunos5():
cflags.remove('-fvisibility=hidden')
-elif platform == 'windows':
+elif platform.is_msvc():
pass
else:
if options.profile == 'gmon':
cflags.append('-pg')
ldflags.append('-pg')
elif options.profile == 'pprof':
- libs.append('-lprofiler')
+ cflags.append('-fno-omit-frame-pointer')
+ libs.extend(['-Wl,--no-as-needed', '-lprofiler'])
def shell_escape(str):
"""Escape str such that it's interpreted as a single argument by
the shell."""
# This isn't complete, but it's just enough to make NINJA_PYTHON work.
- if platform in ('windows', 'mingw'):
+ if platform.is_windows():
return str
if '"' in str:
return "'%s'" % str.replace("'", "\\'")
@@ -189,27 +179,24 @@
n.variable('ldflags', ' '.join(shell_escape(flag) for flag in ldflags))
n.newline()
-if platform == 'windows':
- compiler = '$cxx'
- if options.with_ninja:
- compiler = ('%s -t msvc -o $out -- $cxx /showIncludes' %
- options.with_ninja)
+if platform.is_msvc():
n.rule('cxx',
- command='%s $cflags -c $in /Fo$out' % compiler,
- depfile='$out.d',
- description='CXX $out')
+ command='$cxx /showIncludes $cflags -c $in /Fo$out',
+ description='CXX $out',
+ deps='msvc')
else:
n.rule('cxx',
command='$cxx -MMD -MT $out -MF $out.d $cflags -c $in -o $out',
depfile='$out.d',
+ deps='gcc',
description='CXX $out')
n.newline()
-if host == 'windows':
+if host.is_msvc():
n.rule('ar',
command='lib /nologo /ltcg /out:$out $in',
description='LIB $out')
-elif host == 'mingw':
+elif host.is_mingw():
n.rule('ar',
command='cmd /c $ar cqs $out.tmp $in && move /Y $out.tmp $out',
description='AR $out')
@@ -219,7 +206,7 @@
description='AR $out')
n.newline()
-if platform == 'windows':
+if platform.is_msvc():
n.rule('link',
command='$cxx $in $libs /nologo /link $ldflags /out:$out',
description='LINK $out')
@@ -231,7 +218,7 @@
objs = []
-if platform not in ('solaris', 'mingw', 'windows'):
+if not platform.is_windows() and not platform.is_solaris():
n.comment('browse_py.h is used to inline browse.py.')
n.rule('inline',
command='src/inline.sh $varname < $in > $out',
@@ -269,6 +256,7 @@
'build_log',
'clean',
'depfile_parser',
+ 'deps_log',
'disk_interface',
'edit_distance',
'eval_env',
@@ -276,30 +264,31 @@
'graph',
'graphviz',
'lexer',
+ 'line_printer',
'manifest_parser',
'metrics',
'state',
'util',
'version']:
objs += cxx(name)
-if platform in ('mingw', 'windows'):
+if platform.is_windows():
for name in ['subprocess-win32',
'includes_normalize-win32',
'msvc_helper-win32',
'msvc_helper_main-win32']:
objs += cxx(name)
- if platform == 'windows':
+ if platform.is_msvc():
objs += cxx('minidump-win32')
objs += cc('getopt')
else:
objs += cxx('subprocess-posix')
-if platform == 'windows':
+if platform.is_msvc():
ninja_lib = n.build(built('ninja.lib'), 'ar', objs)
else:
ninja_lib = n.build(built('libninja.a'), 'ar', objs)
n.newline()
-if platform == 'windows':
+if platform.is_msvc():
libs.append('ninja.lib')
else:
libs.append('-lninja')
@@ -324,21 +313,18 @@
path = options.with_gtest
gtest_all_incs = '-I%s -I%s' % (path, os.path.join(path, 'include'))
- if platform == 'windows':
- gtest_cflags = '/nologo /EHsc /Zi ' + gtest_all_incs
+ if platform.is_msvc():
+ gtest_cflags = '/nologo /EHsc /Zi /D_VARIADIC_MAX=10 ' + gtest_all_incs
else:
gtest_cflags = '-fvisibility=hidden ' + gtest_all_incs
objs += n.build(built('gtest-all' + objext), 'cxx',
os.path.join(path, 'src', 'gtest-all.cc'),
variables=[('cflags', gtest_cflags)])
- objs += n.build(built('gtest_main' + objext), 'cxx',
- os.path.join(path, 'src', 'gtest_main.cc'),
- variables=[('cflags', gtest_cflags)])
test_cflags.append('-I%s' % os.path.join(path, 'include'))
else:
# Use gtest from system.
- if platform == 'windows':
+ if platform.is_msvc():
test_libs.extend(['gtest_main.lib', 'gtest.lib'])
else:
test_libs.extend(['-lgtest_main', '-lgtest'])
@@ -348,21 +334,23 @@
'build_test',
'clean_test',
'depfile_parser_test',
+ 'deps_log_test',
'disk_interface_test',
'edit_distance_test',
'graph_test',
'lexer_test',
'manifest_parser_test',
+ 'ninja_test',
'state_test',
'subprocess_test',
'test',
'util_test']:
objs += cxx(name, variables=[('cflags', '$test_cflags')])
-if platform in ('windows', 'mingw'):
+if platform.is_windows():
for name in ['includes_normalize_test', 'msvc_helper_test']:
objs += cxx(name, variables=[('cflags', test_cflags)])
-if platform != 'mingw' and platform != 'windows':
+if not platform.is_windows():
test_libs.append('-lpthread')
ninja_test = n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib,
variables=[('ldflags', test_ldflags),
@@ -425,7 +413,7 @@
implicit=mainpage)
n.newline()
-if host != 'mingw':
+if not host.is_mingw():
n.comment('Regenerate build files if build script changes.')
n.rule('configure',
command='${configure_env}%s configure.py $configure_args' %
@@ -438,7 +426,7 @@
n.default(ninja)
n.newline()
-if host == 'linux':
+if host.is_linux():
n.comment('Packaging')
n.rule('rpmbuild',
command="misc/packaging/rpmbuild.sh",
diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc
index 5b59199..b2c591e 100644
--- a/doc/manual.asciidoc
+++ b/doc/manual.asciidoc
@@ -16,8 +16,8 @@
http://neugierig.org/software/chromium/notes/2011/02/ninja.html[my
work on the Chromium browser project], which has over 30,000 source
files and whose other build systems (including one built from custom
-non-recursive Makefiles) can take ten seconds to start building after
-changing one file. Ninja is under a second.
+non-recursive Makefiles) would take ten seconds to start building
+after changing one file. Ninja is under a second.
Philosophical overview
~~~~~~~~~~~~~~~~~~~~~~
@@ -90,7 +90,7 @@
Comparison to Make
~~~~~~~~~~~~~~~~~~
-Ninja is closest in spirit and functionality to make, relying on
+Ninja is closest in spirit and functionality to Make, relying on
simple dependencies between file timestamps.
But fundamentally, make has a lot of _features_: suffix rules,
@@ -103,13 +103,13 @@
ninja input files. Ninja by itself is unlikely to be useful for most
projects.
-Here are some of the features Ninja adds to make. (These sorts of
+Here are some of the features Ninja adds to Make. (These sorts of
features can often be implemented using more complicated Makefiles,
but they are not part of make itself.)
-* A Ninja rule may point at a path for extra implicit dependency
- information. This makes it easy to get header dependencies correct
- for C/C++ code.
+* Ninja has special support for discovering extra dependencies at build
+ time, making it easy to get <<ref_headers,header dependencies>>
+ correct for C/C++ code.
* A build edge may have multiple outputs.
@@ -137,19 +137,15 @@
Using Ninja for your project
----------------------------
-Ninja currently works on Unix-like systems. It's seen the most testing
-on Linux (and has the best performance there) but it runs fine on Mac
-OS X and FreeBSD. Ninja has some preliminary Windows support but the
-full details of the implementation -- like how to get C header
-interdependencies correct and fast when using MSVC's compiler -- is
-not yet complete.
+Ninja currently works on Unix-like systems and Windows. It's seen the
+most testing on Linux (and has the best performance there) but it runs
+fine on Mac OS X and FreeBSD.
If your project is small, Ninja's speed impact is likely unnoticeable.
-Some build timing numbers are included below. (However, even for
-small projects it sometimes turns out that Ninja's limited syntax
-forces simpler build rules that result in faster builds.) Another way
-to say this is that if you're happy with the edit-compile cycle time
-of your project already then Ninja won't help.
+(However, even for small projects it sometimes turns out that Ninja's
+limited syntax forces simpler build rules that result in faster
+builds.) Another way to say this is that if you're happy with the
+edit-compile cycle time of your project already then Ninja won't help.
There are many other build systems that are more user-friendly or
featureful than Ninja itself. For some recommendations: the Ninja
@@ -161,21 +157,11 @@
meta-build system.
http://code.google.com/p/gyp/[gyp]:: The meta-build system used to
-generate build files for Google Chrome. gyp can generate Ninja files
-for Linux and Mac and is used by many Chrome developers; support for
-Windows is in progress. See the
+generate build files for Google Chrome and related projects (v8,
+node.js). gyp can generate Ninja files for all platforms supported by
+Chrome. See the
http://code.google.com/p/chromium/wiki/NinjaBuild[Chromium Ninja
-documentation for more details]. gyp is relatively unpopular outside
-of the Chrome and v8 world.
-
-* For Chrome (~30k source files), Ninja reduced no-op builds from
- around 15 seconds to under one second.
-* https://plus.google.com/108996039294665965197/posts/SfhrFAhRyyd[A
- Mozilla developer compares build systems]: "While chromium's full
- build is 2.15x slower than firefox's, a nop build is 78.2x faster!
- That is really noticeable during development. No incremental build
- of firefox can be faster than 57.9s, which means that in practice
- almost all of them will be over a minute."
+documentation for more details].
http://www.cmake.org/[CMake]:: A widely used meta-build system that
can generate Ninja files on Linux as of CMake version 2.8.8. (There
@@ -184,11 +170,6 @@
yet officially supported by CMake as the full test suite doesn't
pass.)
-* For building Blender, one user reported "Single file rebuild is 0.97
- sec, same on makefiles was 3.7sec."
-* For building LLVM on Windows, one user reported no-op build times:
- "ninja: 0.4s / MSBuild: 11s / jom: 53s".
-
others:: Ninja ought to fit perfectly into other meta-build software
like http://industriousone.com/premake[premake]. If you do this work,
please let us know!
@@ -222,7 +203,8 @@
`%u`:: The number of remaining edges to start.
`%f`:: The number of finished edges.
`%o`:: Overall rate of finished edges per second
-`%c`:: Current rate of finished edges per second (average over builds specified by -j or its default)
+`%c`:: Current rate of finished edges per second (average over builds
+specified by `-j` or its default)
`%e`:: Elapsed time in seconds. _(Available since Ninja 1.2.)_
`%%`:: A plain `%` character.
@@ -304,8 +286,9 @@
~~~~~~~~~~~~~~~~~~~
Ninja evaluates a graph of dependencies between files, and runs
-whichever commands are necessary to make your build target up to date.
-If you are familiar with Make, Ninja is very similar.
+whichever commands are necessary to make your build target up to date
+as determined by file modification times. If you are familiar with
+Make, Ninja is very similar.
A build file (default name: `build.ninja`) provides a list of _rules_
-- short names for longer commands, like how to run the compiler --
@@ -421,62 +404,6 @@
statement and it is out of date, Ninja will rebuild and reload it
before building the targets requested by the user.
-Pools
-~~~~~
-
-_Available since Ninja 1.1._
-
-Pools allow you to allocate one or more rules or edges a finite number
-of concurrent jobs which is more tightly restricted than the default
-parallelism.
-
-This can be useful, for example, to restrict a particular expensive rule
-(like link steps for huge executables), or to restrict particular build
-statements which you know perform poorly when run concurrently.
-
-Each pool has a `depth` variable which is specified in the build file.
-The pool is then referred to with the `pool` variable on either a rule
-or a build statement.
-
-No matter what pools you specify, ninja will never run more concurrent jobs
-than the default parallelism, or the number of jobs specified on the command
-line (with -j).
-
-----------------
-# No more than 4 links at a time.
-pool link_pool
- depth = 4
-
-# No more than 1 heavy object at a time.
-pool heavy_object_pool
- depth = 1
-
-rule link
- ...
- pool = link_pool
-
-rule cc
- ...
-
-# The link_pool is used here. Only 4 links will run concurrently.
-build foo.exe: link input.obj
-
-# A build statement can be exempted from its rule's pool by setting an
-# empty pool. This effectively puts the build statement back into the default
-# pool, which has infinite depth.
-build other.exe: link input.obj
- pool =
-
-# A build statement can specify a pool directly.
-# Only one of these builds will run at a time.
-build heavy_object1.obj: cc heavy_obj1.cc
- pool = heavy_object_pool
-build heavy_object2.obj: cc heavy_obj2.cc
- pool = heavy_object_pool
-
-----------------
-
-
Generating Ninja files from code
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -581,6 +508,142 @@
come up yet so it's difficult to predict what behavior might be
required.
+[[ref_headers]]
+C/C++ header dependencies
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To get C/C++ header dependencies (or any other build dependency that
+works in a similar way) correct Ninja has some extra functionality.
+
+The problem with headers is that the full list of files that a given
+source file depends on can only be discovered by the compiler:
+different preprocessor defines and include paths cause different files
+to be used. Some compilers can emit this information while building,
+and Ninja can use that to get its dependencies perfect.
+
+Consider: if the file has never been compiled, it must be built anyway,
+generating the header dependencies as a side effect. If any file is
+later modified (even in a way that changes which headers it depends
+on) the modification will cause a rebuild as well, keeping the
+dependencies up to date.
+
+When loading these special dependencies, Ninja implicitly adds extra
+build edges such that it is not an error if the listed dependency is
+missing. This allows you to delete a header file and rebuild without
+the build aborting due to a missing input.
+
+depfile
+^^^^^^^
+
+`gcc` (and other compilers like `clang`) support emitting dependency
+information in the syntax of a Makefile. (Any command that can write
+dependencies in this form can be used, not just `gcc`.)
+
+To bring this information into Ninja requires cooperation. On the
+Ninja side, the `depfile` attribute on the `build` must point to a
+path where this data is written. (Ninja only supports the limited
+subset of the Makefile syntax emitted by compilers.) Then the command
+must know to write dependencies into the `depfile` path.
+Use it like in the following example:
+
+----
+rule cc
+ depfile = $out.d
+ command = gcc -MMD -MF $out.d [other gcc flags here]
+----
+
+The `-MMD` flag to `gcc` tells it to output header dependencies, and
+the `-MF` flag tells it where to write them.
+
+deps
+^^^^
+
+_(Available since Ninja 1.3.)_
+
+It turns out that for large projects (and particularly on Windows,
+where the file system is slow) loading these dependency files on
+startup is slow.
+
+Ninja 1.3 can instead process dependencies just after they're generated
+and save a compacted form of the same information in a Ninja-internal
+database.
+
+Ninja supports this processing in two forms.
+
+1. `deps = gcc` specifies that the tool outputs `gcc`-style dependencies
+ in the form of Makefiles. Adding this to the above example will
+ cause Ninja to process the `depfile` immediately after the
+ compilation finishes, then delete the `.d` file (which is only used
+ as a temporary).
+
+2. `deps = msvc` specifies that the tool outputs header dependencies
+ in the form produced by Visual Studio's compiler's
+ http://msdn.microsoft.com/en-us/library/hdkef6tk(v=vs.90).aspx[`/showIncludes`
+ flag]. Briefly, this means the tool outputs specially-formatted lines
+ to its stdout. Ninja then filters these lines from the displayed
+ output. No `depfile` attribute is necessary.
++
+----
+rule cc
+ deps = msvc
+ command = cl /showIncludes -c $in /Fo$out
+----
+
+Pools
+~~~~~
+
+_Available since Ninja 1.1._
+
+Pools allow you to allocate one or more rules or edges a finite number
+of concurrent jobs which is more tightly restricted than the default
+parallelism.
+
+This can be useful, for example, to restrict a particular expensive rule
+(like link steps for huge executables), or to restrict particular build
+statements which you know perform poorly when run concurrently.
+
+Each pool has a `depth` variable which is specified in the build file.
+The pool is then referred to with the `pool` variable on either a rule
+or a build statement.
+
+No matter what pools you specify, ninja will never run more concurrent jobs
+than the default parallelism, or the number of jobs specified on the command
+line (with `-j`).
+
+----------------
+# No more than 4 links at a time.
+pool link_pool
+ depth = 4
+
+# No more than 1 heavy object at a time.
+pool heavy_object_pool
+ depth = 1
+
+rule link
+ ...
+ pool = link_pool
+
+rule cc
+ ...
+
+# The link_pool is used here. Only 4 links will run concurrently.
+build foo.exe: link input.obj
+
+# A build statement can be exempted from its rule's pool by setting an
+# empty pool. This effectively puts the build statement back into the default
+# pool, which has infinite depth.
+build other.exe: link input.obj
+ pool =
+
+# A build statement can specify a pool directly.
+# Only one of these builds will run at a time.
+build heavy_object1.obj: cc heavy_obj1.cc
+ pool = heavy_object_pool
+build heavy_object2.obj: cc heavy_obj2.cc
+ pool = heavy_object_pool
+
+----------------
+
Ninja file reference
--------------------
@@ -667,6 +730,7 @@
considered part of its parent's scope; if it is indented less than the
previous one, it closes the previous scope.
+[[ref_toplevel]]
Top-level variables
~~~~~~~~~~~~~~~~~~~
@@ -696,22 +760,14 @@
`depfile`:: path to an optional `Makefile` that contains extra
_implicit dependencies_ (see <<ref_dependencies,the reference on
- dependency types>>). This is explicitly to support `gcc` and its `-M`
- family of flags, which output the list of headers a given `.c` file
- depends on.
-+
-Use it like in the following example:
-+
-----
-rule cc
- depfile = $out.d
- command = gcc -MMD -MF $out.d [other gcc flags here]
-----
-+
-When loading a `depfile`, Ninja implicitly adds edges such that it is
-not an error if the listed dependency is missing. This allows you to
-delete a depfile-discovered header file and rebuild, without the build
-aborting due to a missing input.
+ dependency types>>). This is explicitly to support C/C++ header
+ dependencies; see <<ref_headers,the full discussion>>.
+
+`deps`:: _(Available since Ninja 1.3.)_ if present, must be one of
+ `gcc` or `msvc` to specify special dependency processing. See
+ <<ref_headers,the full discussion>>. The generated database is
+ stored as `.ninja_deps` in the `builddir`, see <<ref_toplevel,the
+ discussion of `builddir`>>.
`description`:: a short description of the command, used to pretty-print
the command as it's running. The `-v` flag controls whether to print
@@ -822,7 +878,7 @@
----
rule demo
- command = echo "this is a demo of $foo'
+ command = echo "this is a demo of $foo"
build out: demo
foo = bar
diff --git a/doc/style.css b/doc/style.css
index fc22ec1..5d14a1c 100644
--- a/doc/style.css
+++ b/doc/style.css
@@ -1,9 +1,8 @@
body {
margin: 5ex 10ex;
- max-width: 40em;
- line-height: 1.4;
+ max-width: 80ex;
+ line-height: 1.5;
font-family: sans-serif;
- font-size: 0.8em;
}
h1, h2, h3 {
font-weight: normal;
@@ -21,8 +20,15 @@
code {
color: #007;
}
-.chapter {
+div.chapter {
margin-top: 4em;
+ border-top: solid 2px black;
+}
+.section .title {
+ font-size: 1.3em;
+}
+.section .section .title {
+ font-size: 1.2em;
}
p {
margin-top: 0;
diff --git a/misc/bash-completion b/misc/bash-completion
index b40136e..2d6975b 100644
--- a/misc/bash-completion
+++ b/misc/bash-completion
@@ -18,16 +18,23 @@
_ninja_target() {
local cur targets dir line targets_command OPTIND
cur="${COMP_WORDS[COMP_CWORD]}"
- dir="."
- line=$(echo ${COMP_LINE} | cut -d" " -f 2-)
- while getopts C: opt "${line[@]}"; do
- case $opt in
- C) dir="$OPTARG" ;;
- esac
- done;
- targets_command="ninja -C ${dir} -t targets all"
- targets=$((${targets_command} 2>/dev/null) | awk -F: '{print $1}')
- COMPREPLY=($(compgen -W "$targets" -- "$cur"))
- return 0
+
+ if [[ "$cur" == "--"* ]]; then
+ # there is currently only one argument that takes --
+ COMPREPLY=($(compgen -P '--' -W 'version' -- "${cur:2}"))
+ else
+ dir="."
+ line=$(echo ${COMP_LINE} | cut -d" " -f 2-)
+ # filter out all non relevant arguments but keep C for dirs
+ while getopts C:f:j:l:k:nvd:t: opt "${line[@]}"; do
+ case $opt in
+ C) dir="$OPTARG" ;;
+ esac
+ done;
+ targets_command="ninja -C ${dir} -t targets all"
+ targets=$((${targets_command} 2>/dev/null) | awk -F: '{print $1}')
+ COMPREPLY=($(compgen -W "$targets" -- "$cur"))
+ fi
+ return
}
complete -F _ninja_target ninja
diff --git a/misc/ninja.vim b/misc/ninja.vim
index 841902f..d813267 100644
--- a/misc/ninja.vim
+++ b/misc/ninja.vim
@@ -2,9 +2,9 @@
" Language: ninja build file as described at
" http://martine.github.com/ninja/manual.html
" Version: 1.3
-" Last Change: 2012/12/14
+" Last Change: 2013/04/16
" Maintainer: Nicolas Weber <nicolasweber@gmx.de>
-" Version 1.2 of this script is in the upstream vim repository and will be
+" Version 1.3 of this script is in the upstream vim repository and will be
" included in the next vim release. If you change this, please send your change
" upstream.
@@ -16,6 +16,9 @@
finish
endif
+let s:cpo_save = &cpo
+set cpo&vim
+
syn case match
syn match ninjaComment /#.*/ contains=@Spell
@@ -36,7 +39,7 @@
" let assignments.
" manifest_parser.cc, ParseRule()
syn region ninjaRule start="^rule" end="^\ze\S" contains=ALL transparent
-syn keyword ninjaRuleCommand contained command depfile description generator
+syn keyword ninjaRuleCommand contained command deps depfile description generator
\ pool restat rspfile rspfile_content
syn region ninjaPool start="^pool" end="^\ze\S" contains=ALL transparent
@@ -73,3 +76,6 @@
hi def link ninjaVar Identifier
let b:current_syntax = "ninja"
+
+let &cpo = s:cpo_save
+unlet s:cpo_save
diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py
index ece7eb5..d69e3e4 100644
--- a/misc/ninja_syntax.py
+++ b/misc/ninja_syntax.py
@@ -38,7 +38,7 @@
def rule(self, name, command, description=None, depfile=None,
generator=False, pool=None, restat=False, rspfile=None,
- rspfile_content=None):
+ rspfile_content=None, deps=None):
self._line('rule %s' % name)
self.variable('command', command, indent=1)
if description:
@@ -55,6 +55,8 @@
self.variable('rspfile', rspfile, indent=1)
if rspfile_content:
self.variable('rspfile_content', rspfile_content, indent=1)
+ if deps:
+ self.variable('deps', deps, indent=1)
def build(self, outputs, rule, inputs=None, implicit=None, order_only=None,
variables=None):
diff --git a/misc/ninja_test.py b/misc/ninja_syntax_test.py
similarity index 100%
rename from misc/ninja_test.py
rename to misc/ninja_syntax_test.py
diff --git a/platform_helper.py b/platform_helper.py
new file mode 100644
index 0000000..97827c3
--- /dev/null
+++ b/platform_helper.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# Copyright 2011 Google Inc.
+# Copyright 2013 Patrick von Reth <vonreth@kde.org>
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+def platforms():
+ return ['linux', 'darwin', 'freebsd', 'openbsd', 'solaris', 'sunos5',
+ 'mingw', 'msvc']
+
+class Platform( object ):
+ def __init__( self, platform):
+ self._platform = platform
+ if not self._platform is None:
+ return
+ self._platform = sys.platform
+ if self._platform.startswith('linux'):
+ self._platform = 'linux'
+ elif self._platform.startswith('freebsd'):
+ self._platform = 'freebsd'
+ elif self._platform.startswith('openbsd'):
+ self._platform = 'openbsd'
+ elif self._platform.startswith('solaris'):
+ self._platform = 'solaris'
+ elif self._platform.startswith('mingw'):
+ self._platform = 'mingw'
+ elif self._platform.startswith('win'):
+ self._platform = 'msvc'
+
+
+ def platform(self):
+ return self._platform
+
+ def is_linux(self):
+ return self._platform == 'linux'
+
+ def is_mingw(self):
+ return self._platform == 'mingw'
+
+ def is_msvc(self):
+ return self._platform == 'msvc'
+
+ def is_windows(self):
+ return self.is_mingw() or self.is_msvc()
+
+ def is_solaris(self):
+ return self._platform == 'solaris'
+
+ def is_freebsd(self):
+ return self._platform == 'freebsd'
+
+ def is_openbsd(self):
+ return self._platform == 'openbsd'
+
+ def is_sunos5(self):
+ return self._platform == 'sunos5'
diff --git a/src/build.cc b/src/build.cc
index ae47a50..5cf9d27 100644
--- a/src/build.cc
+++ b/src/build.cc
@@ -15,25 +15,21 @@
#include "build.h"
#include <assert.h>
+#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <functional>
-#ifdef _WIN32
-#include <windows.h>
-#else
-#include <unistd.h>
-#include <sys/ioctl.h>
-#include <sys/time.h>
-#endif
-
#if defined(__SVR4) && defined(__sun)
#include <sys/termios.h>
#endif
#include "build_log.h"
+#include "depfile_parser.h"
+#include "deps_log.h"
#include "disk_interface.h"
#include "graph.h"
+#include "msvc_helper.h"
#include "state.h"
#include "subprocess.h"
#include "util.h"
@@ -47,7 +43,7 @@
// Overridden from CommandRunner:
virtual bool CanRunMore();
virtual bool StartCommand(Edge* edge);
- virtual Edge* WaitForCommand(ExitStatus* status, string* /* output */);
+ virtual bool WaitForCommand(Result* result);
private:
queue<Edge*> finished_;
@@ -62,16 +58,14 @@
return true;
}
-Edge* DryRunCommandRunner::WaitForCommand(ExitStatus* status,
- string* /*output*/) {
- if (finished_.empty()) {
- *status = ExitFailure;
- return NULL;
- }
- *status = ExitSuccess;
- Edge* edge = finished_.front();
+bool DryRunCommandRunner::WaitForCommand(Result* result) {
+ if (finished_.empty())
+ return false;
+
+ result->status = ExitSuccess;
+ result->edge = finished_.front();
finished_.pop();
- return edge;
+ return true;
}
} // namespace
@@ -80,25 +74,12 @@
: config_(config),
start_time_millis_(GetTimeMillis()),
started_edges_(0), finished_edges_(0), total_edges_(0),
- have_blank_line_(true), progress_status_format_(NULL),
+ progress_status_format_(NULL),
overall_rate_(), current_rate_(config.parallelism) {
-#ifndef _WIN32
- const char* term = getenv("TERM");
- smart_terminal_ = isatty(1) && term && string(term) != "dumb";
-#else
- // Disable output buffer. It'd be nice to use line buffering but
- // MSDN says: "For some systems, [_IOLBF] provides line
- // buffering. However, for Win32, the behavior is the same as _IOFBF
- // - Full Buffering."
- setvbuf(stdout, NULL, _IONBF, 0);
- console_ = GetStdHandle(STD_OUTPUT_HANDLE);
- CONSOLE_SCREEN_BUFFER_INFO csbi;
- smart_terminal_ = GetConsoleScreenBufferInfo(console_, &csbi);
-#endif
// Don't do anything fancy in verbose mode.
if (config_.verbosity != BuildConfig::NORMAL)
- smart_terminal_ = false;
+ printer_.set_smart_terminal(false);
progress_status_format_ = getenv("NINJA_STATUS");
if (!progress_status_format_)
@@ -133,17 +114,14 @@
if (config_.verbosity == BuildConfig::QUIET)
return;
- if (smart_terminal_)
+ if (printer_.is_smart_terminal())
PrintStatus(edge);
- if (!success || !output.empty()) {
- if (smart_terminal_)
- printf("\n");
+ // Print the command that is spewing before printing its output.
+ if (!success)
+ printer_.PrintOnNewLine("FAILED: " + edge->EvaluateCommand() + "\n");
- // Print the command that is spewing before printing its output.
- if (!success)
- printf("FAILED: %s\n", edge->EvaluateCommand().c_str());
-
+ if (!output.empty()) {
// ninja sets stdout and stderr of subprocesses to a pipe, to be able to
// check if the output is empty. Some compilers, e.g. clang, check
// isatty(stderr) to decide if they should print colored output.
@@ -157,21 +135,16 @@
// thousands of parallel compile commands.)
// TODO: There should be a flag to disable escape code stripping.
string final_output;
- if (!smart_terminal_)
+ if (!printer_.is_smart_terminal())
final_output = StripAnsiEscapeCodes(output);
else
final_output = output;
-
- if (!final_output.empty())
- printf("%s", final_output.c_str());
-
- have_blank_line_ = true;
+ printer_.PrintOnNewLine(final_output);
}
}
void BuildStatus::BuildFinished() {
- if (smart_terminal_ && !have_blank_line_)
- printf("\n");
+ printer_.PrintOnNewLine("");
}
string BuildStatus::FormatProgressStatus(
@@ -267,72 +240,14 @@
if (to_print.empty() || force_full_command)
to_print = edge->GetBinding("command");
-#ifdef _WIN32
- CONSOLE_SCREEN_BUFFER_INFO csbi;
- GetConsoleScreenBufferInfo(console_, &csbi);
-#endif
-
- if (smart_terminal_) {
-#ifndef _WIN32
- printf("\r"); // Print over previous line, if any.
-#else
- csbi.dwCursorPosition.X = 0;
- SetConsoleCursorPosition(console_, csbi.dwCursorPosition);
-#endif
- }
-
if (finished_edges_ == 0) {
overall_rate_.Restart();
current_rate_.Restart();
}
to_print = FormatProgressStatus(progress_status_format_) + to_print;
- if (smart_terminal_ && !force_full_command) {
-#ifndef _WIN32
- // Limit output to width of the terminal if provided so we don't cause
- // line-wrapping.
- winsize size;
- if ((ioctl(0, TIOCGWINSZ, &size) == 0) && size.ws_col) {
- to_print = ElideMiddle(to_print, size.ws_col);
- }
-#else
- // Don't use the full width or console will move to next line.
- size_t width = static_cast<size_t>(csbi.dwSize.X) - 1;
- to_print = ElideMiddle(to_print, width);
-#endif
- }
-
- if (smart_terminal_ && !force_full_command) {
-#ifndef _WIN32
- printf("%s", to_print.c_str());
- printf("\x1B[K"); // Clear to end of line.
- fflush(stdout);
- have_blank_line_ = false;
-#else
- // We don't want to have the cursor spamming back and forth, so
- // use WriteConsoleOutput instead which updates the contents of
- // the buffer, but doesn't move the cursor position.
- GetConsoleScreenBufferInfo(console_, &csbi);
- COORD buf_size = { csbi.dwSize.X, 1 };
- COORD zero_zero = { 0, 0 };
- SMALL_RECT target = { csbi.dwCursorPosition.X, csbi.dwCursorPosition.Y,
- (SHORT)(csbi.dwCursorPosition.X + csbi.dwSize.X - 1),
- csbi.dwCursorPosition.Y };
- CHAR_INFO* char_data = new CHAR_INFO[csbi.dwSize.X];
- memset(char_data, 0, sizeof(CHAR_INFO) * csbi.dwSize.X);
- for (int i = 0; i < csbi.dwSize.X; ++i) {
- char_data[i].Char.AsciiChar = ' ';
- char_data[i].Attributes = csbi.wAttributes;
- }
- for (size_t i = 0; i < to_print.size(); ++i)
- char_data[i].Char.AsciiChar = to_print[i];
- WriteConsoleOutput(console_, char_data, buf_size, zero_zero, &target);
- delete[] char_data;
- have_blank_line_ = false;
-#endif
- } else {
- printf("%s\n", to_print.c_str());
- }
+ printer_.Print(to_print,
+ force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE);
}
Plan::Plan() : command_edges_(0), wanted_edges_(0) {}
@@ -514,7 +429,7 @@
if (!(*ni)->dirty())
continue;
- if (scan->RecomputeOutputDirty(*ei, most_recent_input,
+ if (scan->RecomputeOutputDirty(*ei, most_recent_input, 0,
command, *ni)) {
(*ni)->MarkDirty();
all_outputs_clean = false;
@@ -549,7 +464,7 @@
virtual ~RealCommandRunner() {}
virtual bool CanRunMore();
virtual bool StartCommand(Edge* edge);
- virtual Edge* WaitForCommand(ExitStatus* status, string* output);
+ virtual bool WaitForCommand(Result* result);
virtual vector<Edge*> GetActiveEdges();
virtual void Abort();
@@ -586,31 +501,30 @@
return true;
}
-Edge* RealCommandRunner::WaitForCommand(ExitStatus* status, string* output) {
+bool RealCommandRunner::WaitForCommand(Result* result) {
Subprocess* subproc;
while ((subproc = subprocs_.NextFinished()) == NULL) {
bool interrupted = subprocs_.DoWork();
- if (interrupted) {
- *status = ExitInterrupted;
- return 0;
- }
+ if (interrupted)
+ return false;
}
- *status = subproc->Finish();
- *output = subproc->GetOutput();
+ result->status = subproc->Finish();
+ result->output = subproc->GetOutput();
map<Subprocess*, Edge*>::iterator i = subproc_to_edge_.find(subproc);
- Edge* edge = i->second;
+ result->edge = i->second;
subproc_to_edge_.erase(i);
delete subproc;
- return edge;
+ return true;
}
Builder::Builder(State* state, const BuildConfig& config,
- BuildLog* log, DiskInterface* disk_interface)
+ BuildLog* build_log, DepsLog* deps_log,
+ DiskInterface* disk_interface)
: state_(state), config_(config), disk_interface_(disk_interface),
- scan_(state, log, disk_interface) {
+ scan_(state, build_log, deps_log, disk_interface) {
status_ = new BuildStatus(config);
}
@@ -696,8 +610,6 @@
// First, we attempt to start as many commands as allowed by the
// command runner.
// Second, we attempt to wait for / reap the next finished command.
- // If we can do neither of those, the build is stuck, and we report
- // an error.
while (plan_.more_to_do()) {
// See if we can start any more commands.
if (failures_allowed && command_runner_->CanRunMore()) {
@@ -707,10 +619,11 @@
return false;
}
- if (edge->is_phony())
- FinishEdge(edge, true, "");
- else
+ if (edge->is_phony()) {
+ plan_.EdgeFinished(edge);
+ } else {
++pending_commands;
+ }
// We made some progress; go back to the main loop.
continue;
@@ -719,34 +632,24 @@
// See if we can reap any finished commands.
if (pending_commands) {
- ExitStatus status;
- string output;
- Edge* edge = command_runner_->WaitForCommand(&status, &output);
- if (edge && status != ExitInterrupted) {
- bool success = (status == ExitSuccess);
- --pending_commands;
- FinishEdge(edge, success, output);
- if (!success) {
- if (failures_allowed)
- failures_allowed--;
- }
-
- // We made some progress; start the main loop over.
- continue;
- }
-
- if (status == ExitInterrupted) {
+ CommandRunner::Result result;
+ if (!command_runner_->WaitForCommand(&result) ||
+ result.status == ExitInterrupted) {
status_->BuildFinished();
*err = "interrupted by user";
return false;
}
- }
- // If we get here, we can neither enqueue new commands nor are any running.
- if (pending_commands) {
- status_->BuildFinished();
- *err = "stuck: pending commands but none to wait for? [this is a bug]";
- return false;
+ --pending_commands;
+ FinishCommand(&result);
+
+ if (!result.success()) {
+ if (failures_allowed)
+ failures_allowed--;
+ }
+
+ // We made some progress; start the main loop over.
+ continue;
}
// If we get here, we cannot make any more progress.
@@ -801,63 +704,145 @@
return true;
}
-void Builder::FinishEdge(Edge* edge, bool success, const string& output) {
- METRIC_RECORD("FinishEdge");
+void Builder::FinishCommand(CommandRunner::Result* result) {
+ METRIC_RECORD("FinishCommand");
+
+ Edge* edge = result->edge;
+
+ // First try to extract dependencies from the result, if any.
+ // This must happen first as it filters the command output (we want
+ // to filter /showIncludes output, even on compile failure) and
+ // extraction itself can fail, which makes the command fail from a
+ // build perspective.
+ vector<Node*> deps_nodes;
+ string deps_type = edge->GetBinding("deps");
+ if (!deps_type.empty()) {
+ string extract_err;
+ if (!ExtractDeps(result, deps_type, &deps_nodes, &extract_err) &&
+ result->success()) {
+ if (!result->output.empty())
+ result->output.append("\n");
+ result->output.append(extract_err);
+ result->status = ExitFailure;
+ }
+ }
+
+ int start_time, end_time;
+ status_->BuildEdgeFinished(edge, result->success(), result->output,
+ &start_time, &end_time);
+
+ // The rest of this function only applies to successful commands.
+ if (!result->success())
+ return;
+
+ // Restat the edge outputs, if necessary.
TimeStamp restat_mtime = 0;
+ if (edge->GetBindingBool("restat") && !config_.dry_run) {
+ bool node_cleaned = false;
- if (success) {
- if (edge->GetBindingBool("restat") && !config_.dry_run) {
- bool node_cleaned = false;
-
- for (vector<Node*>::iterator i = edge->outputs_.begin();
- i != edge->outputs_.end(); ++i) {
- TimeStamp new_mtime = disk_interface_->Stat((*i)->path());
- if ((*i)->mtime() == new_mtime) {
- // The rule command did not change the output. Propagate the clean
- // state through the build graph.
- // Note that this also applies to nonexistent outputs (mtime == 0).
- plan_.CleanNode(&scan_, *i);
- node_cleaned = true;
- }
- }
-
- if (node_cleaned) {
- // If any output was cleaned, find the most recent mtime of any
- // (existing) non-order-only input or the depfile.
- for (vector<Node*>::iterator i = edge->inputs_.begin();
- i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
- TimeStamp input_mtime = disk_interface_->Stat((*i)->path());
- if (input_mtime > restat_mtime)
- restat_mtime = input_mtime;
- }
-
- string depfile = edge->GetBinding("depfile");
- if (restat_mtime != 0 && !depfile.empty()) {
- TimeStamp depfile_mtime = disk_interface_->Stat(depfile);
- if (depfile_mtime > restat_mtime)
- restat_mtime = depfile_mtime;
- }
-
- // The total number of edges in the plan may have changed as a result
- // of a restat.
- status_->PlanHasTotalEdges(plan_.command_edge_count());
+ for (vector<Node*>::iterator i = edge->outputs_.begin();
+ i != edge->outputs_.end(); ++i) {
+ TimeStamp new_mtime = disk_interface_->Stat((*i)->path());
+ if ((*i)->mtime() == new_mtime) {
+ // The rule command did not change the output. Propagate the clean
+ // state through the build graph.
+ // Note that this also applies to nonexistent outputs (mtime == 0).
+ plan_.CleanNode(&scan_, *i);
+ node_cleaned = true;
}
}
- // Delete the response file on success (if exists)
- string rspfile = edge->GetBinding("rspfile");
- if (!rspfile.empty())
- disk_interface_->RemoveFile(rspfile);
+ if (node_cleaned) {
+ // If any output was cleaned, find the most recent mtime of any
+ // (existing) non-order-only input or the depfile.
+ for (vector<Node*>::iterator i = edge->inputs_.begin();
+ i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
+ TimeStamp input_mtime = disk_interface_->Stat((*i)->path());
+ if (input_mtime > restat_mtime)
+ restat_mtime = input_mtime;
+ }
- plan_.EdgeFinished(edge);
+ string depfile = edge->GetBinding("depfile");
+ if (restat_mtime != 0 && !depfile.empty()) {
+ TimeStamp depfile_mtime = disk_interface_->Stat(depfile);
+ if (depfile_mtime > restat_mtime)
+ restat_mtime = depfile_mtime;
+ }
+
+ // The total number of edges in the plan may have changed as a result
+ // of a restat.
+ status_->PlanHasTotalEdges(plan_.command_edge_count());
+ }
}
- if (edge->is_phony())
- return;
+ plan_.EdgeFinished(edge);
- int start_time, end_time;
- status_->BuildEdgeFinished(edge, success, output, &start_time, &end_time);
- if (success && scan_.build_log())
- scan_.build_log()->RecordCommand(edge, start_time, end_time, restat_mtime);
+ // Delete any left over response file.
+ string rspfile = edge->GetBinding("rspfile");
+ if (!rspfile.empty())
+ disk_interface_->RemoveFile(rspfile);
+
+ if (scan_.build_log()) {
+ scan_.build_log()->RecordCommand(edge, start_time, end_time,
+ restat_mtime);
+ }
+
+ if (!deps_type.empty() && !config_.dry_run) {
+ assert(edge->outputs_.size() == 1 && "should have been rejected by parser");
+ Node* out = edge->outputs_[0];
+ TimeStamp deps_mtime = disk_interface_->Stat(out->path());
+ scan_.deps_log()->RecordDeps(out, deps_mtime, deps_nodes);
+ }
+
}
+bool Builder::ExtractDeps(CommandRunner::Result* result,
+ const string& deps_type,
+ vector<Node*>* deps_nodes,
+ string* err) {
+#ifdef _WIN32
+ if (deps_type == "msvc") {
+ CLParser parser;
+ result->output = parser.Parse(result->output);
+ for (set<string>::iterator i = parser.includes_.begin();
+ i != parser.includes_.end(); ++i) {
+ deps_nodes->push_back(state_->GetNode(*i));
+ }
+ } else
+#endif
+ if (deps_type == "gcc") {
+ string depfile = result->edge->GetBinding("depfile");
+ if (depfile.empty()) {
+ *err = string("edge with deps=gcc but no depfile makes no sense");
+ return false;
+ }
+
+ string content = disk_interface_->ReadFile(depfile, err);
+ if (!err->empty())
+ return false;
+ if (content.empty())
+ return true;
+
+ DepfileParser deps;
+ if (!deps.Parse(&content, err))
+ return false;
+
+ // XXX check depfile matches expected output.
+ deps_nodes->reserve(deps.ins_.size());
+ for (vector<StringPiece>::iterator i = deps.ins_.begin();
+ i != deps.ins_.end(); ++i) {
+ if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, err))
+ return false;
+ deps_nodes->push_back(state_->GetNode(*i));
+ }
+
+ if (disk_interface_->RemoveFile(depfile) < 0) {
+ *err = string("deleting depfile: ") + strerror(errno) + string("\n");
+ return false;
+ }
+ } else {
+ Fatal("unknown deps type '%s'", deps_type.c_str());
+ }
+
+ return true;
+}
diff --git a/src/build.h b/src/build.h
index 5747170..2715c0c 100644
--- a/src/build.h
+++ b/src/build.h
@@ -25,6 +25,7 @@
#include "graph.h" // XXX needed for DependencyScan; should rearrange.
#include "exit_status.h"
+#include "line_printer.h"
#include "metrics.h"
#include "util.h" // int64_t
@@ -103,8 +104,18 @@
virtual ~CommandRunner() {}
virtual bool CanRunMore() = 0;
virtual bool StartCommand(Edge* edge) = 0;
- /// Wait for a command to complete.
- virtual Edge* WaitForCommand(ExitStatus* status, string* output) = 0;
+
+ /// The result of waiting for a command.
+ struct Result {
+ Result() : edge(NULL) {}
+ Edge* edge;
+ ExitStatus status;
+ string output;
+ bool success() const { return status == ExitSuccess; }
+ };
+ /// Wait for a command to complete, or return false if interrupted.
+ virtual bool WaitForCommand(Result* result) = 0;
+
virtual vector<Edge*> GetActiveEdges() { return vector<Edge*>(); }
virtual void Abort() {}
};
@@ -131,7 +142,8 @@
/// Builder wraps the build process: starting commands, updating status.
struct Builder {
Builder(State* state, const BuildConfig& config,
- BuildLog* log, DiskInterface* disk_interface);
+ BuildLog* build_log, DepsLog* deps_log,
+ DiskInterface* disk_interface);
~Builder();
/// Clean up after interrupted commands by deleting output files.
@@ -151,7 +163,7 @@
bool Build(string* err);
bool StartEdge(Edge* edge, string* err);
- void FinishEdge(Edge* edge, bool success, const string& output);
+ void FinishCommand(CommandRunner::Result* result);
/// Used for tests.
void SetBuildLog(BuildLog* log) {
@@ -165,6 +177,9 @@
BuildStatus* status_;
private:
+ bool ExtractDeps(CommandRunner::Result* result, const string& deps_type,
+ vector<Node*>* deps_nodes, string* err);
+
DiskInterface* disk_interface_;
DependencyScan scan_;
@@ -198,14 +213,12 @@
int started_edges_, finished_edges_, total_edges_;
- bool have_blank_line_;
-
/// Map of running edge to time the edge started running.
typedef map<Edge*, int> RunningEdgeMap;
RunningEdgeMap running_edges_;
- /// Whether we can do fancy terminal control codes.
- bool smart_terminal_;
+ /// Prints progress output.
+ LinePrinter printer_;
/// The custom progress status format to use.
const char* progress_status_format_;
@@ -255,17 +268,12 @@
double rate_;
Stopwatch stopwatch_;
const size_t N;
- std::queue<double> times_;
+ queue<double> times_;
int last_update_;
};
mutable RateInfo overall_rate_;
mutable SlidingRateInfo current_rate_;
-
-#ifdef _WIN32
- void* console_;
-#endif
};
#endif // NINJA_BUILD_H_
-
diff --git a/src/build_log.h b/src/build_log.h
index 231bfd9..6eae89f 100644
--- a/src/build_log.h
+++ b/src/build_log.h
@@ -15,7 +15,6 @@
#ifndef NINJA_BUILD_LOG_H_
#define NINJA_BUILD_LOG_H_
-#include <map>
#include <string>
#include <stdio.h>
using namespace std;
diff --git a/src/build_log_test.cc b/src/build_log_test.cc
index 2dd6500..4639bc9 100644
--- a/src/build_log_test.cc
+++ b/src/build_log_test.cc
@@ -143,15 +143,7 @@
log2.RecordCommand(state_.edges_[1], 20, 25);
log2.Close();
-#ifndef _WIN32
- ASSERT_EQ(0, truncate(kTestFilename, size));
-#else
- int fh;
- fh = _sopen(kTestFilename, _O_RDWR | _O_CREAT, _SH_DENYNO,
- _S_IREAD | _S_IWRITE);
- ASSERT_EQ(0, _chsize(fh, size));
- _close(fh);
-#endif
+ ASSERT_TRUE(Truncate(kTestFilename, size, &err));
BuildLog log3;
err.clear();
diff --git a/src/build_test.cc b/src/build_test.cc
index 40a82ed..90c328a 100644
--- a/src/build_test.cc
+++ b/src/build_test.cc
@@ -15,6 +15,7 @@
#include "build.h"
#include "build_log.h"
+#include "deps_log.h"
#include "graph.h"
#include "test.h"
@@ -23,6 +24,26 @@
// to create Nodes and Edges.
struct PlanTest : public StateTestWithBuiltinRules {
Plan plan_;
+
+ /// Because FindWork does not return Edges in any sort of predictable order,
+ // provide a means to get available Edges in order and in a format which is
+ // easy to write tests around.
+ void FindWorkSorted(deque<Edge*>* ret, int count) {
+ struct CompareEdgesByOutput {
+ static bool cmp(const Edge* a, const Edge* b) {
+ return a->outputs_[0]->path() < b->outputs_[0]->path();
+ }
+ };
+
+ for (int i = 0; i < count; ++i) {
+ ASSERT_TRUE(plan_.more_to_do());
+ Edge* edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ret->push_back(edge);
+ }
+ ASSERT_FALSE(plan_.FindWork());
+ sort(ret->begin(), ret->end(), CompareEdgesByOutput::cmp);
+ }
};
TEST_F(PlanTest, Basic) {
@@ -241,8 +262,8 @@
));
// Mark all the out* nodes dirty
for (int i = 0; i < 3; ++i) {
- GetNode("out" + string(1, '1' + i))->MarkDirty();
- GetNode("outb" + string(1, '1' + i))->MarkDirty();
+ GetNode("out" + string(1, '1' + static_cast<char>(i)))->MarkDirty();
+ GetNode("outb" + string(1, '1' + static_cast<char>(i)))->MarkDirty();
}
GetNode("allTheThings")->MarkDirty();
@@ -250,27 +271,21 @@
EXPECT_TRUE(plan_.AddTarget(GetNode("allTheThings"), &err));
ASSERT_EQ("", err);
- // Grab the first 4 edges, out1 out2 outb1 outb2
deque<Edge*> edges;
+ FindWorkSorted(&edges, 5);
+
for (int i = 0; i < 4; ++i) {
- ASSERT_TRUE(plan_.more_to_do());
- Edge* edge = plan_.FindWork();
- ASSERT_TRUE(edge);
+ Edge *edge = edges[i];
ASSERT_EQ("in", edge->inputs_[0]->path());
string base_name(i < 2 ? "out" : "outb");
ASSERT_EQ(base_name + string(1, '1' + (i % 2)), edge->outputs_[0]->path());
- edges.push_back(edge);
}
// outb3 is exempt because it has an empty pool
- ASSERT_TRUE(plan_.more_to_do());
- Edge* edge = plan_.FindWork();
+ Edge* edge = edges[4];
ASSERT_TRUE(edge);
ASSERT_EQ("in", edge->inputs_[0]->path());
ASSERT_EQ("outb3", edge->outputs_[0]->path());
- edges.push_back(edge);
-
- ASSERT_FALSE(plan_.FindWork());
// finish out1
plan_.EdgeFinished(edges.front());
@@ -292,11 +307,11 @@
plan_.EdgeFinished(*it);
}
- Edge* final = plan_.FindWork();
- ASSERT_TRUE(final);
- ASSERT_EQ("allTheThings", final->outputs_[0]->path());
+ Edge* last = plan_.FindWork();
+ ASSERT_TRUE(last);
+ ASSERT_EQ("allTheThings", last->outputs_[0]->path());
- plan_.EdgeFinished(final);
+ plan_.EdgeFinished(last);
ASSERT_FALSE(plan_.more_to_do());
ASSERT_FALSE(plan_.FindWork());
@@ -333,25 +348,28 @@
Edge* edge = NULL;
- edge = plan_.FindWork();
- ASSERT_TRUE(edge);
+ deque<Edge*> initial_edges;
+ FindWorkSorted(&initial_edges, 2);
+
+ edge = initial_edges[1]; // Foo first
ASSERT_EQ("foo.cpp", edge->outputs_[0]->path());
plan_.EdgeFinished(edge);
edge = plan_.FindWork();
ASSERT_TRUE(edge);
+ ASSERT_FALSE(plan_.FindWork());
ASSERT_EQ("foo.cpp", edge->inputs_[0]->path());
ASSERT_EQ("foo.cpp", edge->inputs_[1]->path());
ASSERT_EQ("foo.cpp.obj", edge->outputs_[0]->path());
plan_.EdgeFinished(edge);
- edge = plan_.FindWork();
- ASSERT_TRUE(edge);
+ edge = initial_edges[0]; // Now for bar
ASSERT_EQ("bar.cpp", edge->outputs_[0]->path());
plan_.EdgeFinished(edge);
edge = plan_.FindWork();
ASSERT_TRUE(edge);
+ ASSERT_FALSE(plan_.FindWork());
ASSERT_EQ("bar.cpp", edge->inputs_[0]->path());
ASSERT_EQ("bar.cpp", edge->inputs_[1]->path());
ASSERT_EQ("bar.cpp.obj", edge->outputs_[0]->path());
@@ -359,6 +377,7 @@
edge = plan_.FindWork();
ASSERT_TRUE(edge);
+ ASSERT_FALSE(plan_.FindWork());
ASSERT_EQ("foo.cpp.obj", edge->inputs_[0]->path());
ASSERT_EQ("bar.cpp.obj", edge->inputs_[1]->path());
ASSERT_EQ("libfoo.a", edge->outputs_[0]->path());
@@ -366,6 +385,7 @@
edge = plan_.FindWork();
ASSERT_TRUE(edge);
+ ASSERT_FALSE(plan_.FindWork());
ASSERT_EQ("libfoo.a", edge->inputs_[0]->path());
ASSERT_EQ("all", edge->outputs_[0]->path());
plan_.EdgeFinished(edge);
@@ -375,20 +395,40 @@
ASSERT_FALSE(plan_.more_to_do());
}
+/// Fake implementation of CommandRunner, useful for tests.
+struct FakeCommandRunner : public CommandRunner {
+ explicit FakeCommandRunner(VirtualFileSystem* fs) :
+ last_command_(NULL), fs_(fs) {}
-struct BuildTest : public StateTestWithBuiltinRules,
- public CommandRunner {
- BuildTest() : config_(MakeConfig()),
- builder_(&state_, config_, NULL, &fs_),
- now_(1), last_command_(NULL), status_(config_) {
- builder_.command_runner_.reset(this);
+ // CommandRunner impl
+ virtual bool CanRunMore();
+ virtual bool StartCommand(Edge* edge);
+ virtual bool WaitForCommand(Result* result);
+ virtual vector<Edge*> GetActiveEdges();
+ virtual void Abort();
+
+ vector<string> commands_ran_;
+ Edge* last_command_;
+ VirtualFileSystem* fs_;
+};
+
+struct BuildTest : public StateTestWithBuiltinRules {
+ BuildTest() : config_(MakeConfig()), command_runner_(&fs_),
+ builder_(&state_, config_, NULL, NULL, &fs_),
+ status_(config_) {
+ }
+
+ virtual void SetUp() {
+ StateTestWithBuiltinRules::SetUp();
+
+ builder_.command_runner_.reset(&command_runner_);
AssertParse(&state_,
"build cat1: cat in1\n"
"build cat2: cat in1 in2\n"
"build cat12: cat cat1 cat2\n");
- fs_.Create("in1", now_, "");
- fs_.Create("in2", now_, "");
+ fs_.Create("in1", "");
+ fs_.Create("in2", "");
}
~BuildTest() {
@@ -398,13 +438,6 @@
// Mark a path dirty.
void Dirty(const string& path);
- // CommandRunner impl
- virtual bool CanRunMore();
- virtual bool StartCommand(Edge* edge);
- virtual Edge* WaitForCommand(ExitStatus* status, string* output);
- virtual vector<Edge*> GetActiveEdges();
- virtual void Abort();
-
BuildConfig MakeConfig() {
BuildConfig config;
config.verbosity = BuildConfig::QUIET;
@@ -412,31 +445,19 @@
}
BuildConfig config_;
+ FakeCommandRunner command_runner_;
VirtualFileSystem fs_;
Builder builder_;
- int now_;
- vector<string> commands_ran_;
- Edge* last_command_;
BuildStatus status_;
};
-void BuildTest::Dirty(const string& path) {
- Node* node = GetNode(path);
- node->MarkDirty();
-
- // If it's an input file, mark that we've already stat()ed it and
- // it's missing.
- if (!node->in_edge())
- node->MarkMissing();
-}
-
-bool BuildTest::CanRunMore() {
+bool FakeCommandRunner::CanRunMore() {
// Only run one at a time.
return last_command_ == NULL;
}
-bool BuildTest::StartCommand(Edge* edge) {
+bool FakeCommandRunner::StartCommand(Edge* edge) {
assert(!last_command_);
commands_ran_.push_back(edge->EvaluateCommand());
if (edge->rule().name() == "cat" ||
@@ -446,7 +467,7 @@
edge->rule().name() == "touch-interrupt") {
for (vector<Node*>::iterator out = edge->outputs_.begin();
out != edge->outputs_.end(); ++out) {
- fs_.Create((*out)->path(), now_, "");
+ fs_->Create((*out)->path(), "");
}
} else if (edge->rule().name() == "true" ||
edge->rule().name() == "fail" ||
@@ -461,36 +482,48 @@
return true;
}
-Edge* BuildTest::WaitForCommand(ExitStatus* status, string* /* output */) {
- if (Edge* edge = last_command_) {
- if (edge->rule().name() == "interrupt" ||
- edge->rule().name() == "touch-interrupt") {
- *status = ExitInterrupted;
- return NULL;
- }
+bool FakeCommandRunner::WaitForCommand(Result* result) {
+ if (!last_command_)
+ return false;
- if (edge->rule().name() == "fail")
- *status = ExitFailure;
- else
- *status = ExitSuccess;
- last_command_ = NULL;
- return edge;
+ Edge* edge = last_command_;
+ result->edge = edge;
+
+ if (edge->rule().name() == "interrupt" ||
+ edge->rule().name() == "touch-interrupt") {
+ result->status = ExitInterrupted;
+ return true;
}
- *status = ExitFailure;
- return NULL;
+
+ if (edge->rule().name() == "fail")
+ result->status = ExitFailure;
+ else
+ result->status = ExitSuccess;
+ last_command_ = NULL;
+ return true;
}
-vector<Edge*> BuildTest::GetActiveEdges() {
+vector<Edge*> FakeCommandRunner::GetActiveEdges() {
vector<Edge*> edges;
if (last_command_)
edges.push_back(last_command_);
return edges;
}
-void BuildTest::Abort() {
+void FakeCommandRunner::Abort() {
last_command_ = NULL;
}
+void BuildTest::Dirty(const string& path) {
+ Node* node = GetNode(path);
+ node->MarkDirty();
+
+ // If it's an input file, mark that we've already stat()ed it and
+ // it's missing.
+ if (!node->in_edge())
+ node->MarkMissing();
+}
+
TEST_F(BuildTest, NoWork) {
string err;
EXPECT_TRUE(builder_.AlreadyUpToDate());
@@ -506,8 +539,8 @@
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- ASSERT_EQ(1u, commands_ran_.size());
- EXPECT_EQ("cat in1 > cat1", commands_ran_[0]);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cat in1 > cat1", command_runner_.commands_ran_[0]);
}
TEST_F(BuildTest, OneStep2) {
@@ -520,8 +553,8 @@
EXPECT_TRUE(builder_.Build(&err));
EXPECT_EQ("", err);
- ASSERT_EQ(1u, commands_ran_.size());
- EXPECT_EQ("cat in1 > cat1", commands_ran_[0]);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cat in1 > cat1", command_runner_.commands_ran_[0]);
}
TEST_F(BuildTest, TwoStep) {
@@ -530,29 +563,29 @@
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
EXPECT_EQ("", err);
- ASSERT_EQ(3u, commands_ran_.size());
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
// Depending on how the pointers work out, we could've ran
// the first two commands in either order.
- EXPECT_TRUE((commands_ran_[0] == "cat in1 > cat1" &&
- commands_ran_[1] == "cat in1 in2 > cat2") ||
- (commands_ran_[1] == "cat in1 > cat1" &&
- commands_ran_[0] == "cat in1 in2 > cat2"));
+ EXPECT_TRUE((command_runner_.commands_ran_[0] == "cat in1 > cat1" &&
+ command_runner_.commands_ran_[1] == "cat in1 in2 > cat2") ||
+ (command_runner_.commands_ran_[1] == "cat in1 > cat1" &&
+ command_runner_.commands_ran_[0] == "cat in1 in2 > cat2"));
- EXPECT_EQ("cat cat1 cat2 > cat12", commands_ran_[2]);
+ EXPECT_EQ("cat cat1 cat2 > cat12", command_runner_.commands_ran_[2]);
- now_++;
+ fs_.Tick();
// Modifying in2 requires rebuilding one intermediate file
// and the final file.
- fs_.Create("in2", now_, "");
+ fs_.Create("in2", "");
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("cat12", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- ASSERT_EQ(5u, commands_ran_.size());
- EXPECT_EQ("cat in1 in2 > cat2", commands_ran_[3]);
- EXPECT_EQ("cat cat1 cat2 > cat12", commands_ran_[4]);
+ ASSERT_EQ(5u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cat in1 in2 > cat2", command_runner_.commands_ran_[3]);
+ EXPECT_EQ("cat cat1 cat2 > cat12", command_runner_.commands_ran_[4]);
}
TEST_F(BuildTest, TwoOutputs) {
@@ -561,15 +594,15 @@
" command = touch $out\n"
"build out1 out2: touch in.txt\n"));
- fs_.Create("in.txt", now_, "");
+ fs_.Create("in.txt", "");
string err;
EXPECT_TRUE(builder_.AddTarget("out1", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
EXPECT_EQ("", err);
- ASSERT_EQ(1u, commands_ran_.size());
- EXPECT_EQ("touch out1 out2", commands_ran_[0]);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("touch out1 out2", command_runner_.commands_ran_[0]);
}
// Test case from
@@ -581,8 +614,9 @@
"build in1 otherfile: touch in\n"
"build out: touch in | in1\n"));
- fs_.Create("in", now_, "");
- fs_.Create("in1", ++now_, "");
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.Create("in1", "");
string err;
EXPECT_TRUE(builder_.AddTarget("out", &err));
@@ -598,33 +632,33 @@
"build c4: cat c3\n"
"build c5: cat c4\n"));
- fs_.Create("c1", now_, "");
+ fs_.Create("c1", "");
string err;
EXPECT_TRUE(builder_.AddTarget("c5", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
EXPECT_EQ("", err);
- ASSERT_EQ(4u, commands_ran_.size());
+ ASSERT_EQ(4u, command_runner_.commands_ran_.size());
err.clear();
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("c5", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.AlreadyUpToDate());
- now_++;
+ fs_.Tick();
- fs_.Create("c3", now_, "");
+ fs_.Create("c3", "");
err.clear();
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("c5", &err));
ASSERT_EQ("", err);
EXPECT_FALSE(builder_.AlreadyUpToDate());
EXPECT_TRUE(builder_.Build(&err));
- ASSERT_EQ(2u, commands_ran_.size()); // 3->4, 4->5
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // 3->4, 4->5
}
TEST_F(BuildTest, MissingInput) {
@@ -657,7 +691,6 @@
#endif
EXPECT_EQ("", err);
- now_ = 0; // Make all stat()s return file not found.
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
ASSERT_EQ(2u, fs_.directories_made_.size());
@@ -674,7 +707,7 @@
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"rule cc\n command = cc $in\n depfile = $out.d\n"
"build foo.o: cc foo.c\n"));
- fs_.Create("foo.c", now_, "");
+ fs_.Create("foo.c", "");
EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
ASSERT_EQ("", err);
@@ -690,9 +723,9 @@
"build foo.o: cc foo.c\n"));
Edge* edge = state_.edges_.back();
- fs_.Create("foo.c", now_, "");
+ fs_.Create("foo.c", "");
GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing.
- fs_.Create("foo.o.d", now_, "foo.o: blah.h bar.h\n");
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
ASSERT_EQ("", err);
ASSERT_EQ(1u, fs_.files_read_.size());
@@ -713,8 +746,8 @@
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"rule cc\n command = cc $in\n depfile = $out.d\n"
"build foo.o: cc foo.c\n"));
- fs_.Create("foo.c", now_, "");
- fs_.Create("foo.o.d", now_, "randomtext\n");
+ fs_.Create("foo.c", "");
+ fs_.Create("foo.o.d", "randomtext\n");
EXPECT_FALSE(builder_.AddTarget("foo.o", &err));
EXPECT_EQ("expected depfile 'foo.o.d' to mention 'foo.o', got 'randomtext'",
err);
@@ -727,9 +760,9 @@
"build foo.o: cc foo.c || otherfile\n"));
Edge* edge = state_.edges_.back();
- fs_.Create("foo.c", now_, "");
- fs_.Create("otherfile", now_, "");
- fs_.Create("foo.o.d", now_, "foo.o: blah.h bar.h\n");
+ fs_.Create("foo.c", "");
+ fs_.Create("otherfile", "");
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
ASSERT_EQ("", err);
@@ -750,25 +783,31 @@
// explicit dep dirty, expect a rebuild.
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- ASSERT_EQ(1u, commands_ran_.size());
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
- now_++;
+ fs_.Tick();
+
+ // Recreate the depfile, as it should have been deleted by the build.
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
// implicit dep dirty, expect a rebuild.
- fs_.Create("blah.h", now_, "");
- fs_.Create("bar.h", now_, "");
- commands_ran_.clear();
+ fs_.Create("blah.h", "");
+ fs_.Create("bar.h", "");
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- ASSERT_EQ(1u, commands_ran_.size());
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
- now_++;
+ fs_.Tick();
+
+ // Recreate the depfile, as it should have been deleted by the build.
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
// order only dep dirty, no rebuild.
- fs_.Create("otherfile", now_, "");
- commands_ran_.clear();
+ fs_.Create("otherfile", "");
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
EXPECT_EQ("", err);
@@ -776,12 +815,12 @@
// implicit dep missing, expect rebuild.
fs_.RemoveFile("bar.h");
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- ASSERT_EQ(1u, commands_ran_.size());
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
}
TEST_F(BuildTest, RebuildOrderOnlyDeps) {
@@ -792,17 +831,17 @@
"build oo.h: cc oo.h.in\n"
"build foo.o: cc foo.c || oo.h\n"));
- fs_.Create("foo.c", now_, "");
- fs_.Create("oo.h.in", now_, "");
+ fs_.Create("foo.c", "");
+ fs_.Create("oo.h.in", "");
// foo.o and order-only dep dirty, build both.
EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- ASSERT_EQ(2u, commands_ran_.size());
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
// all clean, no rebuild.
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
EXPECT_EQ("", err);
@@ -810,25 +849,25 @@
// order-only dep missing, build it only.
fs_.RemoveFile("oo.h");
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- ASSERT_EQ(1u, commands_ran_.size());
- ASSERT_EQ("cc oo.h.in", commands_ran_[0]);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ ASSERT_EQ("cc oo.h.in", command_runner_.commands_ran_[0]);
- now_++;
+ fs_.Tick();
// order-only dep dirty, build it only.
- fs_.Create("oo.h.in", now_, "");
- commands_ran_.clear();
+ fs_.Create("oo.h.in", "");
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- ASSERT_EQ(1u, commands_ran_.size());
- ASSERT_EQ("cc oo.h.in", commands_ran_[0]);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ ASSERT_EQ("cc oo.h.in", command_runner_.commands_ran_[0]);
}
TEST_F(BuildTest, Phony) {
@@ -836,7 +875,7 @@
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build out: cat bar.cc\n"
"build all: phony out\n"));
- fs_.Create("bar.cc", now_, "");
+ fs_.Create("bar.cc", "");
EXPECT_TRUE(builder_.AddTarget("all", &err));
ASSERT_EQ("", err);
@@ -845,7 +884,7 @@
EXPECT_FALSE(builder_.AlreadyUpToDate());
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- ASSERT_EQ(1u, commands_ran_.size());
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
}
TEST_F(BuildTest, PhonyNoWork) {
@@ -853,8 +892,8 @@
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build out: cat bar.cc\n"
"build all: phony out\n"));
- fs_.Create("bar.cc", now_, "");
- fs_.Create("out", now_, "");
+ fs_.Create("bar.cc", "");
+ fs_.Create("out", "");
EXPECT_TRUE(builder_.AddTarget("all", &err));
ASSERT_EQ("", err);
@@ -872,7 +911,7 @@
ASSERT_EQ("", err);
EXPECT_FALSE(builder_.Build(&err));
- ASSERT_EQ(1u, commands_ran_.size());
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
ASSERT_EQ("subcommand failed", err);
}
@@ -893,7 +932,7 @@
ASSERT_EQ("", err);
EXPECT_FALSE(builder_.Build(&err));
- ASSERT_EQ(3u, commands_ran_.size());
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
ASSERT_EQ("subcommands failed", err);
}
@@ -914,7 +953,7 @@
ASSERT_EQ("", err);
EXPECT_FALSE(builder_.Build(&err));
- ASSERT_EQ(3u, commands_ran_.size());
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
ASSERT_EQ("cannot make progress due to previous errors", err);
}
@@ -934,8 +973,8 @@
// Create input/output that would be considered up to date when
// not considering the command line hash.
- fs_.Create("in", now_, "");
- fs_.Create("out1", now_, "");
+ fs_.Create("in", "");
+ fs_.Create("out1", "");
string err;
// Because it's not in the log, it should not be up-to-date until
@@ -943,7 +982,7 @@
EXPECT_TRUE(builder_.AddTarget("out1", &err));
EXPECT_FALSE(builder_.AlreadyUpToDate());
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("out1", &err));
@@ -963,13 +1002,13 @@
"build out2: true out1\n"
"build out3: cat out2\n"));
- fs_.Create("out1", now_, "");
- fs_.Create("out2", now_, "");
- fs_.Create("out3", now_, "");
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+ fs_.Create("out3", "");
- now_++;
+ fs_.Tick();
- fs_.Create("in", now_, "");
+ fs_.Create("in", "");
// Do a pre-build so that there's commands in the log for the outputs,
// otherwise, the lack of an entry in the build log will cause out3 to rebuild
@@ -979,39 +1018,39 @@
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
- now_++;
+ fs_.Tick();
- fs_.Create("in", now_, "");
+ fs_.Create("in", "");
// "cc" touches out1, so we should build out2. But because "true" does not
// touch out2, we should cancel the build of out3.
EXPECT_TRUE(builder_.AddTarget("out3", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
- ASSERT_EQ(2u, commands_ran_.size());
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
// If we run again, it should be a no-op, because the build log has recorded
// that we've already built out2 with an input timestamp of 2 (from out1).
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("out3", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.AlreadyUpToDate());
- now_++;
+ fs_.Tick();
- fs_.Create("in", now_, "");
+ fs_.Create("in", "");
// The build log entry should not, however, prevent us from rebuilding out2
// if out1 changes.
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("out3", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
- ASSERT_EQ(2u, commands_ran_.size());
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
}
TEST_F(BuildWithLogTest, RestatMissingFile) {
@@ -1028,8 +1067,8 @@
"build out1: true in\n"
"build out2: cc out1\n"));
- fs_.Create("in", now_, "");
- fs_.Create("out2", now_, "");
+ fs_.Create("in", "");
+ fs_.Create("out2", "");
// Do a pre-build so that there's commands in the log for the outputs,
// otherwise, the lack of an entry in the build log will cause out2 to rebuild
@@ -1039,12 +1078,12 @@
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
- now_++;
- fs_.Create("in", now_, "");
- fs_.Create("out2", now_, "");
+ fs_.Tick();
+ fs_.Create("in", "");
+ fs_.Create("out2", "");
// Run a build, expect only the first command to run.
// It doesn't touch its output (due to being the "true" command), so
@@ -1052,7 +1091,7 @@
EXPECT_TRUE(builder_.AddTarget("out2", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
- ASSERT_EQ(1u, commands_ran_.size());
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
}
// Test scenario, in which an input file is removed, but output isn't changed
@@ -1069,21 +1108,21 @@
"build out2: cc out1\n"));
// Create all necessary files
- fs_.Create("in", now_, "");
+ fs_.Create("in", "");
// The implicit dependencies and the depfile itself
// are newer than the output
- TimeStamp restat_mtime = ++now_;
- fs_.Create("out1.d", now_, "out1: will.be.deleted restat.file\n");
- fs_.Create("will.be.deleted", now_, "");
- fs_.Create("restat.file", now_, "");
+ TimeStamp restat_mtime = fs_.Tick();
+ fs_.Create("out1.d", "out1: will.be.deleted restat.file\n");
+ fs_.Create("will.be.deleted", "");
+ fs_.Create("restat.file", "");
// Run the build, out1 and out2 get built
string err;
EXPECT_TRUE(builder_.AddTarget("out2", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
- ASSERT_EQ(2u, commands_ran_.size());
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
// See that an entry in the logfile is created, capturing
// the right mtime
@@ -1096,12 +1135,12 @@
fs_.RemoveFile("will.be.deleted");
// Trigger the build again - only out1 gets built
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("out2", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
- ASSERT_EQ(1u, commands_ran_.size());
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
// Check that the logfile entry remains correctly set
log_entry = build_log_.LookupByOutput("out1");
@@ -1127,13 +1166,13 @@
"build out2: true out1\n"
"build out3: cat out2\n"));
- fs_.Create("out1", now_, "");
- fs_.Create("out2", now_, "");
- fs_.Create("out3", now_, "");
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+ fs_.Create("out3", "");
- now_++;
+ fs_.Tick();
- fs_.Create("in", now_, "");
+ fs_.Create("in", "");
// "cc" touches out1, so we should build out2. But because "true" does not
// touch out2, we should cancel the build of out3.
@@ -1141,7 +1180,7 @@
EXPECT_TRUE(builder_.AddTarget("out3", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
- ASSERT_EQ(3u, commands_ran_.size());
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
}
// Test that RSP files are created when & where appropriate and deleted after
@@ -1158,13 +1197,13 @@
" rspfile = out2.rsp\n"
" long_command = Some very long command\n"));
- fs_.Create("out1", now_, "");
- fs_.Create("out2", now_, "");
- fs_.Create("out3", now_, "");
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+ fs_.Create("out3", "");
- now_++;
+ fs_.Tick();
- fs_.Create("in", now_, "");
+ fs_.Create("in", "");
string err;
EXPECT_TRUE(builder_.AddTarget("out1", &err));
@@ -1176,7 +1215,7 @@
size_t files_removed = fs_.files_removed_.size();
EXPECT_TRUE(builder_.Build(&err));
- ASSERT_EQ(2u, commands_ran_.size()); // cat + cat_rsp
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // cat + cat_rsp
// The RSP file was created
ASSERT_EQ(files_created + 1, fs_.files_created_.size());
@@ -1198,9 +1237,9 @@
" rspfile = out.rsp\n"
" long_command = Another very long command\n"));
- fs_.Create("out", now_, "");
- now_++;
- fs_.Create("in", now_, "");
+ fs_.Create("out", "");
+ fs_.Tick();
+ fs_.Create("in", "");
string err;
EXPECT_TRUE(builder_.AddTarget("out", &err));
@@ -1211,7 +1250,7 @@
EXPECT_FALSE(builder_.Build(&err));
ASSERT_EQ("subcommand failed", err);
- ASSERT_EQ(1u, commands_ran_.size());
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
// The RSP file was created
ASSERT_EQ(files_created + 1, fs_.files_created_.size());
@@ -1237,9 +1276,9 @@
" rspfile = out.rsp\n"
" long_command = Original very long command\n"));
- fs_.Create("out", now_, "");
- now_++;
- fs_.Create("in", now_, "");
+ fs_.Create("out", "");
+ fs_.Tick();
+ fs_.Create("in", "");
string err;
EXPECT_TRUE(builder_.AddTarget("out", &err));
@@ -1247,10 +1286,10 @@
// 1. Build for the 1st time (-> populate log)
EXPECT_TRUE(builder_.Build(&err));
- ASSERT_EQ(1u, commands_ran_.size());
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
// 2. Build again (no change)
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("out", &err));
EXPECT_EQ("", err);
@@ -1265,12 +1304,12 @@
log_entry->command_hash));
log_entry->command_hash++; // Change the command hash to something else.
// Now expect the target to be rebuilt
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("out", &err));
EXPECT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
- EXPECT_EQ(1u, commands_ran_.size());
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
}
TEST_F(BuildTest, InterruptCleanup) {
@@ -1282,11 +1321,11 @@
"build out1: interrupt in1\n"
"build out2: touch-interrupt in2\n"));
- fs_.Create("out1", now_, "");
- fs_.Create("out2", now_, "");
- now_++;
- fs_.Create("in1", now_, "");
- fs_.Create("in2", now_, "");
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+ fs_.Tick();
+ fs_.Create("in1", "");
+ fs_.Create("in2", "");
// An untouched output of an interrupted command should be retained.
string err;
@@ -1295,7 +1334,7 @@
EXPECT_FALSE(builder_.Build(&err));
EXPECT_EQ("interrupted by user", err);
builder_.Cleanup();
- EXPECT_EQ(now_-1, fs_.Stat("out1"));
+ EXPECT_GT(fs_.Stat("out1"), 0);
err = "";
// A touched output of an interrupted command should be deleted.
@@ -1312,8 +1351,8 @@
"build nonexistent: phony\n"
"build out1: cat || nonexistent\n"
"build out2: cat nonexistent\n"));
- fs_.Create("out1", now_, "");
- fs_.Create("out2", now_, "");
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
// out1 should be up to date even though its input is dirty, because its
// order-only dependency has nothing to do.
@@ -1324,13 +1363,31 @@
// out2 should still be out of date though, because its input is dirty.
err.clear();
- commands_ran_.clear();
+ command_runner_.commands_ran_.clear();
state_.Reset();
EXPECT_TRUE(builder_.AddTarget("out2", &err));
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
EXPECT_EQ("", err);
- ASSERT_EQ(1u, commands_ran_.size());
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, DepsGccWithEmptyDepfileErrorsOut) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n"
+" command = cc\n"
+" deps = gcc\n"
+"build out: cc\n"));
+ Dirty("out");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_FALSE(builder_.AlreadyUpToDate());
+
+ EXPECT_FALSE(builder_.Build(&err));
+ ASSERT_EQ("subcommand failed", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
}
TEST_F(BuildTest, StatusFormatReplacePlaceholder) {
@@ -1338,3 +1395,210 @@
status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]"));
}
+TEST_F(BuildTest, FailedDepsParse) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build bad_deps.o: cat in1\n"
+" deps = gcc\n"
+" depfile = in1.d\n"));
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("bad_deps.o", &err));
+ ASSERT_EQ("", err);
+
+ // These deps will fail to parse, as they should only have one
+ // path to the left of the colon.
+ fs_.Create("in1.d", "AAA BBB");
+
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+}
+
+/// Tests of builds involving deps logs necessarily must span
+/// multiple builds. We reuse methods on BuildTest but not the
+/// builder_ it sets up, because we want pristine objects for
+/// each build.
+struct BuildWithDepsLogTest : public BuildTest {
+ BuildWithDepsLogTest() {}
+
+ virtual void SetUp() {
+ BuildTest::SetUp();
+
+ temp_dir_.CreateAndEnter("BuildWithDepsLogTest");
+ }
+
+ virtual void TearDown() {
+ temp_dir_.Cleanup();
+ }
+
+ ScopedTempDir temp_dir_;
+
+ /// Shadow parent class builder_ so we don't accidentally use it.
+ void* builder_;
+};
+
+/// Run a straightforwad build where the deps log is used.
+TEST_F(BuildWithDepsLogTest, Straightforward) {
+ string err;
+ // Note: in1 was created by the superclass SetUp().
+ const char* manifest =
+ "build out: cat in1\n"
+ " deps = gcc\n"
+ " depfile = in1.d\n";
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Run the build once, everything should be ok.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ fs_.Create("in1.d", "out: in2");
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // The deps file should have been removed.
+ EXPECT_EQ(0, fs_.Stat("in1.d"));
+ // Recreate it for the next step.
+ fs_.Create("in1.d", "out: in2");
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Touch the file only mentioned in the deps.
+ fs_.Tick();
+ fs_.Create("in2", "");
+
+ // Run the build again.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ command_runner_.commands_ran_.clear();
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // We should have rebuilt the output due to in2 being
+ // out of date.
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+
+ builder.command_runner_.release();
+ }
+}
+
+/// Verify that obsolete dependency info causes a rebuild.
+/// 1) Run a successful build where everything has time t, record deps.
+/// 2) Move input/output to time t+1 -- despite files in alignment,
+/// should still need to rebuild due to deps at older time.
+TEST_F(BuildWithDepsLogTest, ObsoleteDeps) {
+ string err;
+ // Note: in1 was created by the superclass SetUp().
+ const char* manifest =
+ "build out: cat in1\n"
+ " deps = gcc\n"
+ " depfile = in1.d\n";
+ {
+ // Run an ordinary build that gathers dependencies.
+ fs_.Create("in1", "");
+ fs_.Create("in1.d", "out: ");
+
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Run the build once, everything should be ok.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+
+ // Push all files one tick forward so that only the deps are out
+ // of date.
+ fs_.Tick();
+ fs_.Create("in1", "");
+ fs_.Create("out", "");
+
+ // The deps file should have been removed, so no need to timestamp it.
+ EXPECT_EQ(0, fs_.Stat("in1.d"));
+
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ command_runner_.commands_ran_.clear();
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ // Recreate the deps file here because the build expects them to exist.
+ fs_.Create("in1.d", "out: ");
+
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // We should have rebuilt the output due to the deps being
+ // out of date.
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+
+ builder.command_runner_.release();
+ }
+}
+
+TEST_F(BuildWithDepsLogTest, DepsIgnoredInDryRun) {
+ const char* manifest =
+ "build out: cat in1\n"
+ " deps = gcc\n"
+ " depfile = in1.d\n";
+
+ fs_.Create("out", "");
+ fs_.Tick();
+ fs_.Create("in1", "");
+
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // The deps log is NULL in dry runs.
+ config_.dry_run = true;
+ Builder builder(&state, config_, NULL, NULL, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ command_runner_.commands_ran_.clear();
+
+ string err;
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder.Build(&err));
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+
+ builder.command_runner_.release();
+}
diff --git a/src/clean.cc b/src/clean.cc
index 12afb98..5d1974e 100644
--- a/src/clean.cc
+++ b/src/clean.cc
@@ -15,10 +15,7 @@
#include "clean.h"
#include <assert.h>
-#include <errno.h>
#include <stdio.h>
-#include <string.h>
-#include <sys/stat.h>
#include "disk_interface.h"
#include "graph.h"
diff --git a/src/clean_test.cc b/src/clean_test.cc
index 5ed48da..04cff73 100644
--- a/src/clean_test.cc
+++ b/src/clean_test.cc
@@ -31,10 +31,10 @@
"build out1: cat in1\n"
"build in2: cat src2\n"
"build out2: cat in2\n"));
- fs_.Create("in1", 1, "");
- fs_.Create("out1", 1, "");
- fs_.Create("in2", 1, "");
- fs_.Create("out2", 1, "");
+ fs_.Create("in1", "");
+ fs_.Create("out1", "");
+ fs_.Create("in2", "");
+ fs_.Create("out2", "");
Cleaner cleaner(&state_, config_, &fs_);
@@ -61,10 +61,10 @@
"build out1: cat in1\n"
"build in2: cat src2\n"
"build out2: cat in2\n"));
- fs_.Create("in1", 1, "");
- fs_.Create("out1", 1, "");
- fs_.Create("in2", 1, "");
- fs_.Create("out2", 1, "");
+ fs_.Create("in1", "");
+ fs_.Create("out1", "");
+ fs_.Create("in2", "");
+ fs_.Create("out2", "");
config_.dry_run = true;
Cleaner cleaner(&state_, config_, &fs_);
@@ -92,10 +92,10 @@
"build out1: cat in1\n"
"build in2: cat src2\n"
"build out2: cat in2\n"));
- fs_.Create("in1", 1, "");
- fs_.Create("out1", 1, "");
- fs_.Create("in2", 1, "");
- fs_.Create("out2", 1, "");
+ fs_.Create("in1", "");
+ fs_.Create("out1", "");
+ fs_.Create("in2", "");
+ fs_.Create("out2", "");
Cleaner cleaner(&state_, config_, &fs_);
@@ -122,10 +122,10 @@
"build out1: cat in1\n"
"build in2: cat src2\n"
"build out2: cat in2\n"));
- fs_.Create("in1", 1, "");
- fs_.Create("out1", 1, "");
- fs_.Create("in2", 1, "");
- fs_.Create("out2", 1, "");
+ fs_.Create("in1", "");
+ fs_.Create("out1", "");
+ fs_.Create("in2", "");
+ fs_.Create("out2", "");
config_.dry_run = true;
Cleaner cleaner(&state_, config_, &fs_);
@@ -155,10 +155,10 @@
"build out1: cat in1\n"
"build in2: cat_e src2\n"
"build out2: cat in2\n"));
- fs_.Create("in1", 1, "");
- fs_.Create("out1", 1, "");
- fs_.Create("in2", 1, "");
- fs_.Create("out2", 1, "");
+ fs_.Create("in1", "");
+ fs_.Create("out1", "");
+ fs_.Create("in2", "");
+ fs_.Create("out2", "");
Cleaner cleaner(&state_, config_, &fs_);
@@ -187,10 +187,10 @@
"build out1: cat in1\n"
"build in2: cat_e src2\n"
"build out2: cat in2\n"));
- fs_.Create("in1", 1, "");
- fs_.Create("out1", 1, "");
- fs_.Create("in2", 1, "");
- fs_.Create("out2", 1, "");
+ fs_.Create("in1", "");
+ fs_.Create("out1", "");
+ fs_.Create("in2", "");
+ fs_.Create("out2", "");
config_.dry_run = true;
Cleaner cleaner(&state_, config_, &fs_);
@@ -219,15 +219,15 @@
" generator = 1\n"
"build out1: cat in1\n"
"build out2: regen in2\n"));
- fs_.Create("out1", 1, "");
- fs_.Create("out2", 1, "");
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
Cleaner cleaner(&state_, config_, &fs_);
EXPECT_EQ(0, cleaner.CleanAll());
EXPECT_EQ(1, cleaner.cleaned_files_count());
EXPECT_EQ(1u, fs_.files_removed_.size());
- fs_.Create("out1", 1, "");
+ fs_.Create("out1", "");
EXPECT_EQ(0, cleaner.CleanAll(/*generator=*/true));
EXPECT_EQ(2, cleaner.cleaned_files_count());
@@ -240,8 +240,8 @@
" command = cc $in > $out\n"
" depfile = $out.d\n"
"build out1: cc in1\n"));
- fs_.Create("out1", 1, "");
- fs_.Create("out1.d", 1, "");
+ fs_.Create("out1", "");
+ fs_.Create("out1.d", "");
Cleaner cleaner(&state_, config_, &fs_);
EXPECT_EQ(0, cleaner.CleanAll());
@@ -255,8 +255,8 @@
" command = cc $in > $out\n"
" depfile = $out.d\n"
"build out1: cc in1\n"));
- fs_.Create("out1", 1, "");
- fs_.Create("out1.d", 1, "");
+ fs_.Create("out1", "");
+ fs_.Create("out1.d", "");
Cleaner cleaner(&state_, config_, &fs_);
EXPECT_EQ(0, cleaner.CleanTarget("out1"));
@@ -270,8 +270,8 @@
" command = cc $in > $out\n"
" depfile = $out.d\n"
"build out1: cc in1\n"));
- fs_.Create("out1", 1, "");
- fs_.Create("out1.d", 1, "");
+ fs_.Create("out1", "");
+ fs_.Create("out1.d", "");
Cleaner cleaner(&state_, config_, &fs_);
EXPECT_EQ(0, cleaner.CleanRule("cc"));
@@ -288,8 +288,8 @@
"build out1: cc in1\n"
" rspfile = cc1.rsp\n"
" rspfile_content=$in\n"));
- fs_.Create("out1", 1, "");
- fs_.Create("cc1.rsp", 1, "");
+ fs_.Create("out1", "");
+ fs_.Create("cc1.rsp", "");
Cleaner cleaner(&state_, config_, &fs_);
EXPECT_EQ(0, cleaner.CleanAll());
@@ -311,12 +311,12 @@
"build out2: cat_rsp in2\n"
" rspfile=out2.rsp\n"
" rspfile_content=$in\n"));
- fs_.Create("in1", 1, "");
- fs_.Create("out1", 1, "");
- fs_.Create("in2.rsp", 1, "");
- fs_.Create("out2.rsp", 1, "");
- fs_.Create("in2", 1, "");
- fs_.Create("out2", 1, "");
+ fs_.Create("in1", "");
+ fs_.Create("out1", "");
+ fs_.Create("in2.rsp", "");
+ fs_.Create("out2.rsp", "");
+ fs_.Create("in2", "");
+ fs_.Create("out2", "");
Cleaner cleaner(&state_, config_, &fs_);
ASSERT_EQ(0, cleaner.cleaned_files_count());
@@ -354,9 +354,9 @@
"build t1: cat\n"
"build t2: cat\n"));
- fs_.Create("phony", 1, "");
- fs_.Create("t1", 1, "");
- fs_.Create("t2", 1, "");
+ fs_.Create("phony", "");
+ fs_.Create("t1", "");
+ fs_.Create("t2", "");
// Check that CleanAll does not remove "phony".
Cleaner cleaner(&state_, config_, &fs_);
@@ -364,8 +364,8 @@
EXPECT_EQ(2, cleaner.cleaned_files_count());
EXPECT_NE(0, fs_.Stat("phony"));
- fs_.Create("t1", 1, "");
- fs_.Create("t2", 1, "");
+ fs_.Create("t1", "");
+ fs_.Create("t2", "");
// Check that CleanTarget does not remove "phony".
EXPECT_EQ(0, cleaner.CleanTarget("phony"));
diff --git a/src/deps_log.cc b/src/deps_log.cc
new file mode 100644
index 0000000..931cc77
--- /dev/null
+++ b/src/deps_log.cc
@@ -0,0 +1,326 @@
+// Copyright 2012 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "deps_log.h"
+
+#include <assert.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#ifndef _WIN32
+#include <unistd.h>
+#endif
+
+#include "graph.h"
+#include "metrics.h"
+#include "state.h"
+#include "util.h"
+
+// The version is stored as 4 bytes after the signature and also serves as a
+// byte order mark. Signature and version combined are 16 bytes long.
+const char kFileSignature[] = "# ninjadeps\n";
+const int kCurrentVersion = 1;
+
+DepsLog::~DepsLog() {
+ Close();
+}
+
+bool DepsLog::OpenForWrite(const string& path, string* err) {
+ if (needs_recompaction_) {
+ Close();
+ if (!Recompact(path, err))
+ return false;
+ }
+
+ file_ = fopen(path.c_str(), "ab");
+ if (!file_) {
+ *err = strerror(errno);
+ return false;
+ }
+ SetCloseOnExec(fileno(file_));
+
+ // Opening a file in append mode doesn't set the file pointer to the file's
+ // end on Windows. Do that explicitly.
+ fseek(file_, 0, SEEK_END);
+
+ if (ftell(file_) == 0) {
+ if (fwrite(kFileSignature, sizeof(kFileSignature) - 1, 1, file_) < 1) {
+ *err = strerror(errno);
+ return false;
+ }
+ if (fwrite(&kCurrentVersion, 4, 1, file_) < 1) {
+ *err = strerror(errno);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool DepsLog::RecordDeps(Node* node, TimeStamp mtime,
+ const vector<Node*>& nodes) {
+ return RecordDeps(node, mtime, nodes.size(),
+ nodes.empty() ? NULL : (Node**)&nodes.front());
+}
+
+bool DepsLog::RecordDeps(Node* node, TimeStamp mtime,
+ int node_count, Node** nodes) {
+ // Track whether there's any new data to be recorded.
+ bool made_change = false;
+
+ // Assign ids to all nodes that are missing one.
+ if (node->id() < 0) {
+ RecordId(node);
+ made_change = true;
+ }
+ for (int i = 0; i < node_count; ++i) {
+ if (nodes[i]->id() < 0) {
+ RecordId(nodes[i]);
+ made_change = true;
+ }
+ }
+
+ // See if the new data is different than the existing data, if any.
+ if (!made_change) {
+ Deps* deps = GetDeps(node);
+ if (!deps ||
+ deps->mtime != mtime ||
+ deps->node_count != node_count) {
+ made_change = true;
+ } else {
+ for (int i = 0; i < node_count; ++i) {
+ if (deps->nodes[i] != nodes[i]) {
+ made_change = true;
+ break;
+ }
+ }
+ }
+ }
+
+ // Don't write anything if there's no new info.
+ if (!made_change)
+ return true;
+
+ // Update on-disk representation.
+ uint16_t size = 4 * (1 + 1 + (uint16_t)node_count);
+ size |= 0x8000; // Deps record: set high bit.
+ fwrite(&size, 2, 1, file_);
+ int id = node->id();
+ fwrite(&id, 4, 1, file_);
+ int timestamp = mtime;
+ fwrite(×tamp, 4, 1, file_);
+ for (int i = 0; i < node_count; ++i) {
+ id = nodes[i]->id();
+ fwrite(&id, 4, 1, file_);
+ }
+
+ // Update in-memory representation.
+ Deps* deps = new Deps(mtime, node_count);
+ for (int i = 0; i < node_count; ++i)
+ deps->nodes[i] = nodes[i];
+ UpdateDeps(node->id(), deps);
+
+ return true;
+}
+
+void DepsLog::Close() {
+ if (file_)
+ fclose(file_);
+ file_ = NULL;
+}
+
+bool DepsLog::Load(const string& path, State* state, string* err) {
+ METRIC_RECORD(".ninja_deps load");
+ char buf[32 << 10];
+ FILE* f = fopen(path.c_str(), "rb");
+ if (!f) {
+ if (errno == ENOENT)
+ return true;
+ *err = strerror(errno);
+ return false;
+ }
+
+ bool valid_header = true;
+ int version = 0;
+ if (!fgets(buf, sizeof(buf), f) || fread(&version, 4, 1, f) < 1)
+ valid_header = false;
+ if (!valid_header || strcmp(buf, kFileSignature) != 0 ||
+ version != kCurrentVersion) {
+ *err = "bad deps log signature or version; starting over";
+ fclose(f);
+ unlink(path.c_str());
+ // Don't report this as a failure. An empty deps log will cause
+ // us to rebuild the outputs anyway.
+ return true;
+ }
+
+ long offset;
+ bool read_failed = false;
+ int unique_dep_record_count = 0;
+ int total_dep_record_count = 0;
+ for (;;) {
+ offset = ftell(f);
+
+ uint16_t size;
+ if (fread(&size, 2, 1, f) < 1) {
+ if (!feof(f))
+ read_failed = true;
+ break;
+ }
+ bool is_deps = (size >> 15) != 0;
+ size = size & 0x7FFF;
+
+ if (fread(buf, size, 1, f) < 1) {
+ read_failed = true;
+ break;
+ }
+
+ if (is_deps) {
+ assert(size % 4 == 0);
+ int* deps_data = reinterpret_cast<int*>(buf);
+ int out_id = deps_data[0];
+ int mtime = deps_data[1];
+ deps_data += 2;
+ int deps_count = (size / 4) - 2;
+
+ Deps* deps = new Deps(mtime, deps_count);
+ for (int i = 0; i < deps_count; ++i) {
+ assert(deps_data[i] < (int)nodes_.size());
+ assert(nodes_[deps_data[i]]);
+ deps->nodes[i] = nodes_[deps_data[i]];
+ }
+
+ total_dep_record_count++;
+ if (!UpdateDeps(out_id, deps))
+ ++unique_dep_record_count;
+ } else {
+ StringPiece path(buf, size);
+ Node* node = state->GetNode(path);
+ assert(node->id() < 0);
+ node->set_id(nodes_.size());
+ nodes_.push_back(node);
+ }
+ }
+
+ if (read_failed) {
+ // An error occurred while loading; try to recover by truncating the
+ // file to the last fully-read record.
+ if (ferror(f)) {
+ *err = strerror(ferror(f));
+ } else {
+ *err = "premature end of file";
+ }
+ fclose(f);
+
+ if (!Truncate(path.c_str(), offset, err))
+ return false;
+
+ // The truncate succeeded; we'll just report the load error as a
+ // warning because the build can proceed.
+ *err += "; recovering";
+ return true;
+ }
+
+ fclose(f);
+
+ // Rebuild the log if there are too many dead records.
+ int kMinCompactionEntryCount = 1000;
+ int kCompactionRatio = 3;
+ if (total_dep_record_count > kMinCompactionEntryCount &&
+ total_dep_record_count > unique_dep_record_count * kCompactionRatio) {
+ needs_recompaction_ = true;
+ }
+
+ return true;
+}
+
+DepsLog::Deps* DepsLog::GetDeps(Node* node) {
+ // Abort if the node has no id (never referenced in the deps) or if
+ // there's no deps recorded for the node.
+ if (node->id() < 0 || node->id() >= (int)deps_.size())
+ return NULL;
+ return deps_[node->id()];
+}
+
+bool DepsLog::Recompact(const string& path, string* err) {
+ METRIC_RECORD(".ninja_deps recompact");
+ printf("Recompacting deps...\n");
+
+ string temp_path = path + ".recompact";
+
+ // OpenForWrite() opens for append. Make sure it's not appending to a
+ // left-over file from a previous recompaction attempt that crashed somehow.
+ unlink(temp_path.c_str());
+
+ DepsLog new_log;
+ if (!new_log.OpenForWrite(temp_path, err))
+ return false;
+
+ // Clear all known ids so that new ones can be reassigned. The new indices
+ // will refer to the ordering in new_log, not in the current log.
+ for (vector<Node*>::iterator i = nodes_.begin(); i != nodes_.end(); ++i)
+ (*i)->set_id(-1);
+
+ // Write out all deps again.
+ for (int old_id = 0; old_id < (int)deps_.size(); ++old_id) {
+ Deps* deps = deps_[old_id];
+ if (!deps) continue; // If nodes_[old_id] is a leaf, it has no deps.
+
+ if (!new_log.RecordDeps(nodes_[old_id], deps->mtime,
+ deps->node_count, deps->nodes)) {
+ new_log.Close();
+ return false;
+ }
+ }
+
+ new_log.Close();
+
+ // All nodes now have ids that refer to new_log, so steal its data.
+ deps_.swap(new_log.deps_);
+ nodes_.swap(new_log.nodes_);
+
+ if (unlink(path.c_str()) < 0) {
+ *err = strerror(errno);
+ return false;
+ }
+
+ if (rename(temp_path.c_str(), path.c_str()) < 0) {
+ *err = strerror(errno);
+ return false;
+ }
+
+ return true;
+}
+
+bool DepsLog::UpdateDeps(int out_id, Deps* deps) {
+ if (out_id >= (int)deps_.size())
+ deps_.resize(out_id + 1);
+
+ bool delete_old = deps_[out_id] != NULL;
+ if (delete_old)
+ delete deps_[out_id];
+ deps_[out_id] = deps;
+ return delete_old;
+}
+
+bool DepsLog::RecordId(Node* node) {
+ uint16_t size = (uint16_t)node->path().size();
+ fwrite(&size, 2, 1, file_);
+ fwrite(node->path().data(), node->path().size(), 1, file_);
+
+ node->set_id(nodes_.size());
+ nodes_.push_back(node);
+
+ return true;
+}
diff --git a/src/deps_log.h b/src/deps_log.h
new file mode 100644
index 0000000..de0fe63
--- /dev/null
+++ b/src/deps_log.h
@@ -0,0 +1,110 @@
+// Copyright 2012 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_DEPS_LOG_H_
+#define NINJA_DEPS_LOG_H_
+
+#include <string>
+#include <vector>
+using namespace std;
+
+#include <stdio.h>
+
+#include "timestamp.h"
+
+struct Node;
+struct State;
+
+/// As build commands run they can output extra dependency information
+/// (e.g. header dependencies for C source) dynamically. DepsLog collects
+/// that information at build time and uses it for subsequent builds.
+///
+/// The on-disk format is based on two primary design constraints:
+/// - it must be written to as a stream (during the build, which may be
+/// interrupted);
+/// - it can be read all at once on startup. (Alternative designs, where
+/// it contains indexing information, were considered and discarded as
+/// too complicated to implement; if the file is small than reading it
+/// fully on startup is acceptable.)
+/// Here are some stats from the Windows Chrome dependency files, to
+/// help guide the design space. The total text in the files sums to
+/// 90mb so some compression is warranted to keep load-time fast.
+/// There's about 10k files worth of dependencies that reference about
+/// 40k total paths totalling 2mb of unique strings.
+///
+/// Based on these stats, here's the current design.
+/// The file is structured as version header followed by a sequence of records.
+/// Each record is either a path string or a dependency list.
+/// Numbering the path strings in file order gives them dense integer ids.
+/// A dependency list maps an output id to a list of input ids.
+///
+/// Concretely, a record is:
+/// two bytes record length, high bit indicates record type
+/// (implies max record length 32k)
+/// path records contain just the string name of the path
+/// dependency records are an array of 4-byte integers
+/// [output path id, output path mtime, input path id, input path id...]
+/// (The mtime is compared against the on-disk output path mtime
+/// to verify the stored data is up-to-date.)
+/// If two records reference the same output the latter one in the file
+/// wins, allowing updates to just be appended to the file. A separate
+/// repacking step can run occasionally to remove dead records.
+struct DepsLog {
+ DepsLog() : needs_recompaction_(false), file_(NULL) {}
+ ~DepsLog();
+
+ // Writing (build-time) interface.
+ bool OpenForWrite(const string& path, string* err);
+ bool RecordDeps(Node* node, TimeStamp mtime, const vector<Node*>& nodes);
+ bool RecordDeps(Node* node, TimeStamp mtime, int node_count, Node** nodes);
+ void Close();
+
+ // Reading (startup-time) interface.
+ struct Deps {
+ Deps(int mtime, int node_count)
+ : mtime(mtime), node_count(node_count), nodes(new Node*[node_count]) {}
+ ~Deps() { delete [] nodes; }
+ int mtime;
+ int node_count;
+ Node** nodes;
+ };
+ bool Load(const string& path, State* state, string* err);
+ Deps* GetDeps(Node* node);
+
+ /// Rewrite the known log entries, throwing away old data.
+ bool Recompact(const string& path, string* err);
+
+ /// Used for tests.
+ const vector<Node*>& nodes() const { return nodes_; }
+ const vector<Deps*>& deps() const { return deps_; }
+
+ private:
+ // Updates the in-memory representation. Takes ownership of |deps|.
+ // Returns true if a prior deps record was deleted.
+ bool UpdateDeps(int out_id, Deps* deps);
+ // Write a node name record, assigning it an id.
+ bool RecordId(Node* node);
+
+ bool needs_recompaction_;
+ FILE* file_;
+
+ /// Maps id -> Node.
+ vector<Node*> nodes_;
+ /// Maps id -> deps of that id.
+ vector<Deps*> deps_;
+
+ friend struct DepsLogTest;
+};
+
+#endif // NINJA_DEPS_LOG_H_
diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc
new file mode 100644
index 0000000..0591736
--- /dev/null
+++ b/src/deps_log_test.cc
@@ -0,0 +1,383 @@
+// Copyright 2012 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "deps_log.h"
+
+#include "graph.h"
+#include "util.h"
+#include "test.h"
+
+namespace {
+
+const char kTestFilename[] = "DepsLogTest-tempfile";
+
+struct DepsLogTest : public testing::Test {
+ virtual void SetUp() {
+ // In case a crashing test left a stale file behind.
+ unlink(kTestFilename);
+ }
+ virtual void TearDown() {
+ unlink(kTestFilename);
+ }
+};
+
+TEST_F(DepsLogTest, WriteRead) {
+ State state1;
+ DepsLog log1;
+ string err;
+ EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ {
+ vector<Node*> deps;
+ deps.push_back(state1.GetNode("foo.h"));
+ deps.push_back(state1.GetNode("bar.h"));
+ log1.RecordDeps(state1.GetNode("out.o"), 1, deps);
+
+ deps.clear();
+ deps.push_back(state1.GetNode("foo.h"));
+ deps.push_back(state1.GetNode("bar2.h"));
+ log1.RecordDeps(state1.GetNode("out2.o"), 2, deps);
+
+ DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o"));
+ ASSERT_TRUE(log_deps);
+ ASSERT_EQ(1, log_deps->mtime);
+ ASSERT_EQ(2, log_deps->node_count);
+ ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
+ ASSERT_EQ("bar.h", log_deps->nodes[1]->path());
+ }
+
+ log1.Close();
+
+ State state2;
+ DepsLog log2;
+ EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
+ ASSERT_EQ("", err);
+
+ ASSERT_EQ(log1.nodes().size(), log2.nodes().size());
+ for (int i = 0; i < (int)log1.nodes().size(); ++i) {
+ Node* node1 = log1.nodes()[i];
+ Node* node2 = log2.nodes()[i];
+ ASSERT_EQ(i, node1->id());
+ ASSERT_EQ(node1->id(), node2->id());
+ }
+
+ // Spot-check the entries in log2.
+ DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o"));
+ ASSERT_TRUE(log_deps);
+ ASSERT_EQ(2, log_deps->mtime);
+ ASSERT_EQ(2, log_deps->node_count);
+ ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
+ ASSERT_EQ("bar2.h", log_deps->nodes[1]->path());
+}
+
+// Verify that adding the same deps twice doesn't grow the file.
+TEST_F(DepsLogTest, DoubleEntry) {
+ // Write some deps to the file and grab its size.
+ int file_size;
+ {
+ State state;
+ DepsLog log;
+ string err;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ vector<Node*> deps;
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("bar.h"));
+ log.RecordDeps(state.GetNode("out.o"), 1, deps);
+ log.Close();
+
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ file_size = (int)st.st_size;
+ ASSERT_GT(file_size, 0);
+ }
+
+ // Now reload the file, and readd the same deps.
+ {
+ State state;
+ DepsLog log;
+ string err;
+ EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
+
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ vector<Node*> deps;
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("bar.h"));
+ log.RecordDeps(state.GetNode("out.o"), 1, deps);
+ log.Close();
+
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ int file_size_2 = (int)st.st_size;
+ ASSERT_EQ(file_size, file_size_2);
+ }
+}
+
+// Verify that adding the new deps works and can be compacted away.
+TEST_F(DepsLogTest, Recompact) {
+ // Write some deps to the file and grab its size.
+ int file_size;
+ {
+ State state;
+ DepsLog log;
+ string err;
+ ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ vector<Node*> deps;
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("bar.h"));
+ log.RecordDeps(state.GetNode("out.o"), 1, deps);
+
+ deps.clear();
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("baz.h"));
+ log.RecordDeps(state.GetNode("other_out.o"), 1, deps);
+
+ log.Close();
+
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ file_size = (int)st.st_size;
+ ASSERT_GT(file_size, 0);
+ }
+
+ // Now reload the file, and add slighly different deps.
+ int file_size_2;
+ {
+ State state;
+ DepsLog log;
+ string err;
+ ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
+
+ ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ vector<Node*> deps;
+ deps.push_back(state.GetNode("foo.h"));
+ log.RecordDeps(state.GetNode("out.o"), 1, deps);
+ log.Close();
+
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ file_size_2 = (int)st.st_size;
+ // The file should grow to record the new deps.
+ ASSERT_GT(file_size_2, file_size);
+ }
+
+ // Now reload the file, verify the new deps have replaced the old, then
+ // recompact.
+ {
+ State state;
+ DepsLog log;
+ string err;
+ ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
+
+ Node* out = state.GetNode("out.o");
+ DepsLog::Deps* deps = log.GetDeps(out);
+ ASSERT_TRUE(deps);
+ ASSERT_EQ(1, deps->mtime);
+ ASSERT_EQ(1, deps->node_count);
+ ASSERT_EQ("foo.h", deps->nodes[0]->path());
+
+ Node* other_out = state.GetNode("other_out.o");
+ deps = log.GetDeps(other_out);
+ ASSERT_TRUE(deps);
+ ASSERT_EQ(1, deps->mtime);
+ ASSERT_EQ(2, deps->node_count);
+ ASSERT_EQ("foo.h", deps->nodes[0]->path());
+ ASSERT_EQ("baz.h", deps->nodes[1]->path());
+
+ ASSERT_TRUE(log.Recompact(kTestFilename, &err));
+
+ // The in-memory deps graph should still be valid after recompaction.
+ deps = log.GetDeps(out);
+ ASSERT_TRUE(deps);
+ ASSERT_EQ(1, deps->mtime);
+ ASSERT_EQ(1, deps->node_count);
+ ASSERT_EQ("foo.h", deps->nodes[0]->path());
+ ASSERT_EQ(out, log.nodes()[out->id()]);
+
+ deps = log.GetDeps(other_out);
+ ASSERT_TRUE(deps);
+ ASSERT_EQ(1, deps->mtime);
+ ASSERT_EQ(2, deps->node_count);
+ ASSERT_EQ("foo.h", deps->nodes[0]->path());
+ ASSERT_EQ("baz.h", deps->nodes[1]->path());
+ ASSERT_EQ(other_out, log.nodes()[other_out->id()]);
+
+ // The file should have shrunk a bit for the smaller deps.
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ int file_size_3 = (int)st.st_size;
+ ASSERT_LT(file_size_3, file_size_2);
+ }
+}
+
+// Verify that invalid file headers cause a new build.
+TEST_F(DepsLogTest, InvalidHeader) {
+ const char *kInvalidHeaders[] = {
+ "", // Empty file.
+ "# ninjad", // Truncated first line.
+ "# ninjadeps\n", // No version int.
+ "# ninjadeps\n\001\002", // Truncated version int.
+ "# ninjadeps\n\001\002\003\004" // Invalid version int.
+ };
+ for (size_t i = 0; i < sizeof(kInvalidHeaders) / sizeof(kInvalidHeaders[0]);
+ ++i) {
+ FILE* deps_log = fopen(kTestFilename, "wb");
+ ASSERT_TRUE(deps_log != NULL);
+ ASSERT_EQ(
+ strlen(kInvalidHeaders[i]),
+ fwrite(kInvalidHeaders[i], 1, strlen(kInvalidHeaders[i]), deps_log));
+ ASSERT_EQ(0 ,fclose(deps_log));
+
+ string err;
+ DepsLog log;
+ State state;
+ ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
+ EXPECT_EQ("bad deps log signature or version; starting over", err);
+ }
+}
+
+// Simulate what happens when loading a truncated log file.
+TEST_F(DepsLogTest, Truncated) {
+ // Create a file with some entries.
+ {
+ State state;
+ DepsLog log;
+ string err;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ vector<Node*> deps;
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("bar.h"));
+ log.RecordDeps(state.GetNode("out.o"), 1, deps);
+
+ deps.clear();
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("bar2.h"));
+ log.RecordDeps(state.GetNode("out2.o"), 2, deps);
+
+ log.Close();
+ }
+
+ // Get the file size.
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+
+ // Try reloading at truncated sizes.
+ // Track how many nodes/deps were found; they should decrease with
+ // smaller sizes.
+ int node_count = 5;
+ int deps_count = 2;
+ for (int size = (int)st.st_size; size > 0; --size) {
+ string err;
+ ASSERT_TRUE(Truncate(kTestFilename, size, &err));
+
+ State state;
+ DepsLog log;
+ EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
+ if (!err.empty()) {
+ // At some point the log will be so short as to be unparseable.
+ break;
+ }
+
+ ASSERT_GE(node_count, (int)log.nodes().size());
+ node_count = log.nodes().size();
+
+ // Count how many non-NULL deps entries there are.
+ int new_deps_count = 0;
+ for (vector<DepsLog::Deps*>::const_iterator i = log.deps().begin();
+ i != log.deps().end(); ++i) {
+ if (*i)
+ ++new_deps_count;
+ }
+ ASSERT_GE(deps_count, new_deps_count);
+ deps_count = new_deps_count;
+ }
+}
+
+// Run the truncation-recovery logic.
+TEST_F(DepsLogTest, TruncatedRecovery) {
+ // Create a file with some entries.
+ {
+ State state;
+ DepsLog log;
+ string err;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ vector<Node*> deps;
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("bar.h"));
+ log.RecordDeps(state.GetNode("out.o"), 1, deps);
+
+ deps.clear();
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("bar2.h"));
+ log.RecordDeps(state.GetNode("out2.o"), 2, deps);
+
+ log.Close();
+ }
+
+ // Shorten the file, corrupting the last record.
+ struct stat st;
+ ASSERT_EQ(0, stat(kTestFilename, &st));
+ ASSERT_EQ(0, truncate(kTestFilename, st.st_size - 2));
+
+ // Load the file again, add an entry.
+ {
+ State state;
+ DepsLog log;
+ string err;
+ EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
+ ASSERT_EQ("premature end of file; recovering", err);
+ err.clear();
+
+ // The truncated entry should've been discarded.
+ EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o")));
+
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_EQ("", err);
+
+ // Add a new entry.
+ vector<Node*> deps;
+ deps.push_back(state.GetNode("foo.h"));
+ deps.push_back(state.GetNode("bar2.h"));
+ log.RecordDeps(state.GetNode("out2.o"), 3, deps);
+
+ log.Close();
+ }
+
+ // Load the file a third time to verify appending after a mangled
+ // entry doesn't break things.
+ {
+ State state;
+ DepsLog log;
+ string err;
+ EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
+
+ // The truncated entry should exist.
+ DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o"));
+ ASSERT_TRUE(deps);
+ }
+}
+
+} // anonymous namespace
diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index 7c557cd..ee3e99a 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -80,8 +80,10 @@
// MSDN: "Naming Files, Paths, and Namespaces"
// http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
if (!path.empty() && path[0] != '\\' && path.size() > MAX_PATH) {
- Error("Stat(%s): Filename longer than %i characters",
- path.c_str(), MAX_PATH);
+ if (!quiet_) {
+ Error("Stat(%s): Filename longer than %i characters",
+ path.c_str(), MAX_PATH);
+ }
return -1;
}
WIN32_FILE_ATTRIBUTE_DATA attrs;
@@ -89,8 +91,10 @@
DWORD err = GetLastError();
if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND)
return 0;
- Error("GetFileAttributesEx(%s): %s", path.c_str(),
- GetLastErrorString().c_str());
+ if (!quiet_) {
+ Error("GetFileAttributesEx(%s): %s", path.c_str(),
+ GetLastErrorString().c_str());
+ }
return -1;
}
const FILETIME& filetime = attrs.ftLastWriteTime;
@@ -107,7 +111,9 @@
if (stat(path.c_str(), &st) < 0) {
if (errno == ENOENT || errno == ENOTDIR)
return 0;
- Error("stat(%s): %s", path.c_str(), strerror(errno));
+ if (!quiet_) {
+ Error("stat(%s): %s", path.c_str(), strerror(errno));
+ }
return -1;
}
return st.st_mtime;
diff --git a/src/disk_interface.h b/src/disk_interface.h
index 55f8a21..ff1e21c 100644
--- a/src/disk_interface.h
+++ b/src/disk_interface.h
@@ -55,12 +55,16 @@
/// Implementation of DiskInterface that actually hits the disk.
struct RealDiskInterface : public DiskInterface {
+ RealDiskInterface() : quiet_(false) {}
virtual ~RealDiskInterface() {}
virtual TimeStamp Stat(const string& path);
virtual bool MakeDir(const string& path);
virtual bool WriteFile(const string& path, const string& contents);
virtual string ReadFile(const string& path, string* err);
virtual int RemoveFile(const string& path);
+
+ /// Whether to print on errors. Used to make a test quieter.
+ bool quiet_;
};
#endif // NINJA_DISK_INTERFACE_H_
diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc
index c2315c7..55822a6 100644
--- a/src/disk_interface_test.cc
+++ b/src/disk_interface_test.cc
@@ -60,6 +60,7 @@
}
TEST_F(DiskInterfaceTest, StatBadPath) {
+ disk_.quiet_ = true;
#ifdef _WIN32
string bad_path("cc:\\foo");
EXPECT_EQ(-1, disk_.Stat(bad_path));
@@ -67,6 +68,7 @@
string too_long_name(512, 'x');
EXPECT_EQ(-1, disk_.Stat(too_long_name));
#endif
+ disk_.quiet_ = false;
}
TEST_F(DiskInterfaceTest, StatExistingFile) {
@@ -104,7 +106,7 @@
struct StatTest : public StateTestWithBuiltinRules,
public DiskInterface {
- StatTest() : scan_(&state_, NULL, this) {}
+ StatTest() : scan_(&state_, NULL, NULL, this) {}
// DiskInterface implementation.
virtual TimeStamp Stat(const string& path);
diff --git a/src/edit_distance.cc b/src/edit_distance.cc
index 22db4fe..cc4483f 100644
--- a/src/edit_distance.cc
+++ b/src/edit_distance.cc
@@ -32,8 +32,8 @@
int m = s1.len_;
int n = s2.len_;
- std::vector<int> previous(n + 1);
- std::vector<int> current(n + 1);
+ vector<int> previous(n + 1);
+ vector<int> current(n + 1);
for (int i = 0; i <= n; ++i)
previous[i] = i;
diff --git a/src/graph.cc b/src/graph.cc
index b000c48..b245e52 100644
--- a/src/graph.cc
+++ b/src/graph.cc
@@ -19,6 +19,7 @@
#include "build_log.h"
#include "depfile_parser.h"
+#include "deps_log.h"
#include "disk_interface.h"
#include "explain.h"
#include "manifest_parser.h"
@@ -48,6 +49,7 @@
return var == "command" ||
var == "depfile" ||
var == "description" ||
+ var == "deps" ||
var == "generator" ||
var == "pool" ||
var == "restat" ||
@@ -59,15 +61,12 @@
bool dirty = false;
edge->outputs_ready_ = true;
- string depfile = edge->GetBinding("depfile");
- if (!depfile.empty()) {
- if (!LoadDepFile(edge, depfile, err)) {
- if (!err->empty())
- return false;
- EXPLAIN("Edge targets are dirty because depfile '%s' is missing",
- depfile.c_str());
- dirty = true;
- }
+ TimeStamp deps_mtime = 0;
+ if (!dep_loader_.LoadDeps(edge, &deps_mtime, err)) {
+ if (!err->empty())
+ return false;
+ // Failed to load dependency info: rebuild to regenerate it.
+ dirty = true;
}
// Visit all inputs; we're dirty if any of the inputs are dirty.
@@ -114,7 +113,8 @@
for (vector<Node*>::iterator i = edge->outputs_.begin();
i != edge->outputs_.end(); ++i) {
(*i)->StatIfNecessary(disk_interface_);
- if (RecomputeOutputDirty(edge, most_recent_input, command, *i)) {
+ if (RecomputeOutputDirty(edge, most_recent_input, deps_mtime,
+ command, *i)) {
dirty = true;
break;
}
@@ -143,6 +143,7 @@
bool DependencyScan::RecomputeOutputDirty(Edge* edge,
Node* most_recent_input,
+ TimeStamp deps_mtime,
const string& command,
Node* output) {
if (edge->is_phony()) {
@@ -161,28 +162,36 @@
// Dirty if the output is older than the input.
if (most_recent_input && output->mtime() < most_recent_input->mtime()) {
+ TimeStamp output_mtime = output->mtime();
+
// If this is a restat rule, we may have cleaned the output with a restat
// rule in a previous run and stored the most recent input mtime in the
// build log. Use that mtime instead, so that the file will only be
// considered dirty if an input was modified since the previous run.
- TimeStamp most_recent_stamp = most_recent_input->mtime();
+ bool used_restat = false;
if (edge->GetBindingBool("restat") && build_log() &&
(entry = build_log()->LookupByOutput(output->path()))) {
- if (entry->restat_mtime < most_recent_stamp) {
- EXPLAIN("restat of output %s older than most recent input %s "
- "(%d vs %d)",
- output->path().c_str(), most_recent_input->path().c_str(),
- entry->restat_mtime, most_recent_stamp);
- return true;
- }
- } else {
- EXPLAIN("output %s older than most recent input %s (%d vs %d)",
- output->path().c_str(), most_recent_input->path().c_str(),
- output->mtime(), most_recent_stamp);
+ output_mtime = entry->restat_mtime;
+ used_restat = true;
+ }
+
+ if (output_mtime < most_recent_input->mtime()) {
+ EXPLAIN("%soutput %s older than most recent input %s "
+ "(%d vs %d)",
+ used_restat ? "restat of " : "", output->path().c_str(),
+ most_recent_input->path().c_str(),
+ output_mtime, most_recent_input->mtime());
return true;
}
}
+ // Dirty if the output is newer than the deps.
+ if (deps_mtime && output->mtime() > deps_mtime) {
+ EXPLAIN("stored deps info out of date for for %s (%d vs %d)",
+ output->path().c_str(), deps_mtime, output->mtime());
+ return true;
+ }
+
// May also be dirty due to the command changing since the last build.
// But if this is a generator rule, the command changing does not make us
// dirty.
@@ -281,71 +290,6 @@
return !GetBinding(key).empty();
}
-bool DependencyScan::LoadDepFile(Edge* edge, const string& path, string* err) {
- METRIC_RECORD("depfile load");
- string content = disk_interface_->ReadFile(path, err);
- if (!err->empty()) {
- *err = "loading '" + path + "': " + *err;
- return false;
- }
- // On a missing depfile: return false and empty *err.
- if (content.empty())
- return false;
-
- DepfileParser depfile;
- string depfile_err;
- if (!depfile.Parse(&content, &depfile_err)) {
- *err = path + ": " + depfile_err;
- return false;
- }
-
- // Check that this depfile matches the edge's output.
- Node* first_output = edge->outputs_[0];
- StringPiece opath = StringPiece(first_output->path());
- if (opath != depfile.out_) {
- *err = "expected depfile '" + path + "' to mention '" +
- first_output->path() + "', got '" + depfile.out_.AsString() + "'";
- return false;
- }
-
- // Preallocate space in edge->inputs_ to be filled in below.
- edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_,
- depfile.ins_.size(), 0);
- edge->implicit_deps_ += depfile.ins_.size();
- vector<Node*>::iterator implicit_dep =
- edge->inputs_.end() - edge->order_only_deps_ - depfile.ins_.size();
-
- // Add all its in-edges.
- for (vector<StringPiece>::iterator i = depfile.ins_.begin();
- i != depfile.ins_.end(); ++i, ++implicit_dep) {
- if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, err))
- return false;
-
- Node* node = state_->GetNode(*i);
- *implicit_dep = node;
- node->AddOutEdge(edge);
-
- // If we don't have a edge that generates this input already,
- // create one; this makes us not abort if the input is missing,
- // but instead will rebuild in that circumstance.
- if (!node->in_edge()) {
- Edge* phony_edge = state_->AddEdge(&State::kPhonyRule);
- node->set_in_edge(phony_edge);
- phony_edge->outputs_.push_back(node);
-
- // RecomputeDirty might not be called for phony_edge if a previous call
- // to RecomputeDirty had caused the file to be stat'ed. Because previous
- // invocations of RecomputeDirty would have seen this node without an
- // input edge (and therefore ready), we have to set outputs_ready_ to true
- // to avoid a potential stuck build. If we do call RecomputeDirty for
- // this node, it will simply set outputs_ready_ to the correct value.
- phony_edge->outputs_ready_ = true;
- }
- }
-
- return true;
-}
-
void Edge::Dump(const char* prefix) const {
printf("%s[ ", prefix);
for (vector<Node*>::const_iterator i = inputs_.begin();
@@ -387,3 +331,119 @@
(*e)->Dump(" +- ");
}
}
+
+bool ImplicitDepLoader::LoadDeps(Edge* edge, TimeStamp* mtime, string* err) {
+ string deps_type = edge->GetBinding("deps");
+ if (!deps_type.empty()) {
+ if (!LoadDepsFromLog(edge, mtime, err)) {
+ if (!err->empty())
+ return false;
+ EXPLAIN("deps for %s are missing", edge->outputs_[0]->path().c_str());
+ return false;
+ }
+ return true;
+ }
+
+ string depfile = edge->GetBinding("depfile");
+ if (!depfile.empty()) {
+ if (!LoadDepFile(edge, depfile, err)) {
+ if (!err->empty())
+ return false;
+ EXPLAIN("depfile '%s' is missing", depfile.c_str());
+ return false;
+ }
+ return true;
+ }
+
+ // No deps to load.
+ return true;
+}
+
+bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path,
+ string* err) {
+ METRIC_RECORD("depfile load");
+ string content = disk_interface_->ReadFile(path, err);
+ if (!err->empty()) {
+ *err = "loading '" + path + "': " + *err;
+ return false;
+ }
+ // On a missing depfile: return false and empty *err.
+ if (content.empty())
+ return false;
+
+ DepfileParser depfile;
+ string depfile_err;
+ if (!depfile.Parse(&content, &depfile_err)) {
+ *err = path + ": " + depfile_err;
+ return false;
+ }
+
+ // Check that this depfile matches the edge's output.
+ Node* first_output = edge->outputs_[0];
+ StringPiece opath = StringPiece(first_output->path());
+ if (opath != depfile.out_) {
+ *err = "expected depfile '" + path + "' to mention '" +
+ first_output->path() + "', got '" + depfile.out_.AsString() + "'";
+ return false;
+ }
+
+ // Preallocate space in edge->inputs_ to be filled in below.
+ vector<Node*>::iterator implicit_dep =
+ PreallocateSpace(edge, depfile.ins_.size());
+
+ // Add all its in-edges.
+ for (vector<StringPiece>::iterator i = depfile.ins_.begin();
+ i != depfile.ins_.end(); ++i, ++implicit_dep) {
+ if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, err))
+ return false;
+
+ Node* node = state_->GetNode(*i);
+ *implicit_dep = node;
+ node->AddOutEdge(edge);
+ CreatePhonyInEdge(node);
+ }
+
+ return true;
+}
+
+bool ImplicitDepLoader::LoadDepsFromLog(Edge* edge, TimeStamp* deps_mtime,
+ string* err) {
+ DepsLog::Deps* deps = deps_log_->GetDeps(edge->outputs_[0]);
+ if (!deps)
+ return false;
+
+ *deps_mtime = deps->mtime;
+
+ vector<Node*>::iterator implicit_dep =
+ PreallocateSpace(edge, deps->node_count);
+ for (int i = 0; i < deps->node_count; ++i, ++implicit_dep) {
+ *implicit_dep = deps->nodes[i];
+ CreatePhonyInEdge(*implicit_dep);
+ }
+ return true;
+}
+
+vector<Node*>::iterator ImplicitDepLoader::PreallocateSpace(Edge* edge,
+ int count) {
+ edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_,
+ (size_t)count, 0);
+ edge->implicit_deps_ += count;
+ return edge->inputs_.end() - edge->order_only_deps_ - count;
+}
+
+void ImplicitDepLoader::CreatePhonyInEdge(Node* node) {
+ if (node->in_edge())
+ return;
+
+ Edge* phony_edge = state_->AddEdge(&State::kPhonyRule);
+ node->set_in_edge(phony_edge);
+ phony_edge->outputs_.push_back(node);
+
+ // RecomputeDirty might not be called for phony_edge if a previous call
+ // to RecomputeDirty had caused the file to be stat'ed. Because previous
+ // invocations of RecomputeDirty would have seen this node without an
+ // input edge (and therefore ready), we have to set outputs_ready_ to true
+ // to avoid a potential stuck build. If we do call RecomputeDirty for
+ // this node, it will simply set outputs_ready_ to the correct value.
+ phony_edge->outputs_ready_ = true;
+}
diff --git a/src/graph.h b/src/graph.h
index 8b93e29..428ba01 100644
--- a/src/graph.h
+++ b/src/graph.h
@@ -22,8 +22,13 @@
#include "eval_env.h"
#include "timestamp.h"
+struct BuildLog;
struct DiskInterface;
+struct DepsLog;
struct Edge;
+struct Node;
+struct Pool;
+struct State;
/// Information about a node in the dependency graph: the file, whether
/// it's dirty, mtime, etc.
@@ -32,7 +37,8 @@
: path_(path),
mtime_(-1),
dirty_(false),
- in_edge_(NULL) {}
+ in_edge_(NULL),
+ id_(-1) {}
/// Return true if the file exists (mtime_ got a value).
bool Stat(DiskInterface* disk_interface);
@@ -74,6 +80,9 @@
Edge* in_edge() const { return in_edge_; }
void set_in_edge(Edge* edge) { in_edge_ = edge; }
+ int id() const { return id_; }
+ void set_id(int id) { id_ = id; }
+
const vector<Edge*>& out_edges() const { return out_edges_; }
void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); }
@@ -98,6 +107,9 @@
/// All Edges that use this Node as an input.
vector<Edge*> out_edges_;
+
+ /// A dense integer id for the node, assigned and used by DepsLog.
+ int id_;
};
/// An invokable build command and associated metadata (description, etc.).
@@ -121,11 +133,6 @@
map<string, EvalString> bindings_;
};
-struct BuildLog;
-struct Node;
-struct State;
-struct Pool;
-
/// An edge in the dependency graph; links between Nodes using Rules.
struct Edge {
Edge() : rule_(NULL), env_(NULL), outputs_ready_(false), implicit_deps_(0),
@@ -178,13 +185,54 @@
};
+/// ImplicitDepLoader loads implicit dependencies, as referenced via the
+/// "depfile" attribute in build files.
+struct ImplicitDepLoader {
+ ImplicitDepLoader(State* state, DepsLog* deps_log,
+ DiskInterface* disk_interface)
+ : state_(state), disk_interface_(disk_interface), deps_log_(deps_log) {}
+
+ /// Load implicit dependencies for \a edge. May fill in \a mtime with
+ /// the timestamp of the loaded information.
+ /// @return false on error (without filling \a err if info is just missing).
+ bool LoadDeps(Edge* edge, TimeStamp* mtime, string* err);
+
+ DepsLog* deps_log() const {
+ return deps_log_;
+ }
+
+ private:
+ /// Load implicit dependencies for \a edge from a depfile attribute.
+ /// @return false on error (without filling \a err if info is just missing).
+ bool LoadDepFile(Edge* edge, const string& path, string* err);
+
+ /// Load implicit dependencies for \a edge from the DepsLog.
+ /// @return false on error (without filling \a err if info is just missing).
+ bool LoadDepsFromLog(Edge* edge, TimeStamp* mtime, string* err);
+
+ /// Preallocate \a count spaces in the input array on \a edge, returning
+ /// an iterator pointing at the first new space.
+ vector<Node*>::iterator PreallocateSpace(Edge* edge, int count);
+
+ /// If we don't have a edge that generates this input already,
+ /// create one; this makes us not abort if the input is missing,
+ /// but instead will rebuild in that circumstance.
+ void CreatePhonyInEdge(Node* node);
+
+ State* state_;
+ DiskInterface* disk_interface_;
+ DepsLog* deps_log_;
+};
+
+
/// DependencyScan manages the process of scanning the files in a graph
/// and updating the dirty/outputs_ready state of all the nodes and edges.
struct DependencyScan {
- DependencyScan(State* state, BuildLog* build_log,
+ DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log,
DiskInterface* disk_interface)
- : state_(state), build_log_(build_log),
- disk_interface_(disk_interface) {}
+ : build_log_(build_log),
+ disk_interface_(disk_interface),
+ dep_loader_(state, deps_log, disk_interface) {}
/// Examine inputs, outputs, and command lines to judge whether an edge
/// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_|
@@ -195,10 +243,9 @@
/// Recompute whether a given single output should be marked dirty.
/// Returns true if so.
bool RecomputeOutputDirty(Edge* edge, Node* most_recent_input,
+ TimeStamp deps_mtime,
const string& command, Node* output);
- bool LoadDepFile(Edge* edge, const string& path, string* err);
-
BuildLog* build_log() const {
return build_log_;
}
@@ -206,10 +253,14 @@
build_log_ = log;
}
+ DepsLog* deps_log() const {
+ return dep_loader_.deps_log();
+ }
+
private:
- State* state_;
BuildLog* build_log_;
DiskInterface* disk_interface_;
+ ImplicitDepLoader dep_loader_;
};
#endif // NINJA_GRAPH_H_
diff --git a/src/graph_test.cc b/src/graph_test.cc
index 396def4..63d5757 100644
--- a/src/graph_test.cc
+++ b/src/graph_test.cc
@@ -17,7 +17,7 @@
#include "test.h"
struct GraphTest : public StateTestWithBuiltinRules {
- GraphTest() : scan_(&state_, NULL, &fs_) {}
+ GraphTest() : scan_(&state_, NULL, NULL, &fs_) {}
VirtualFileSystem fs_;
DependencyScan scan_;
@@ -26,8 +26,8 @@
TEST_F(GraphTest, MissingImplicit) {
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build out: cat in | implicit\n"));
- fs_.Create("in", 1, "");
- fs_.Create("out", 1, "");
+ fs_.Create("in", "");
+ fs_.Create("out", "");
Edge* edge = GetNode("out")->in_edge();
string err;
@@ -43,9 +43,10 @@
TEST_F(GraphTest, ModifiedImplicit) {
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build out: cat in | implicit\n"));
- fs_.Create("in", 1, "");
- fs_.Create("out", 1, "");
- fs_.Create("implicit", 2, "");
+ fs_.Create("in", "");
+ fs_.Create("out", "");
+ fs_.Tick();
+ fs_.Create("implicit", "");
Edge* edge = GetNode("out")->in_edge();
string err;
@@ -62,10 +63,11 @@
" depfile = $out.d\n"
" command = cat $in > $out\n"
"build out.o: catdep foo.cc\n"));
- fs_.Create("implicit.h", 2, "");
- fs_.Create("foo.cc", 1, "");
- fs_.Create("out.o.d", 1, "out.o: ./foo/../implicit.h\n");
- fs_.Create("out.o", 1, "");
+ fs_.Create("foo.cc", "");
+ fs_.Create("out.o.d", "out.o: ./foo/../implicit.h\n");
+ fs_.Create("out.o", "");
+ fs_.Tick();
+ fs_.Create("implicit.h", "");
Edge* edge = GetNode("out.o")->in_edge();
string err;
@@ -84,11 +86,12 @@
" command = cat $in > $out\n"
"build implicit.h: cat data\n"
"build out.o: catdep foo.cc || implicit.h\n"));
- fs_.Create("data", 2, "");
- fs_.Create("implicit.h", 1, "");
- fs_.Create("foo.cc", 1, "");
- fs_.Create("out.o.d", 1, "out.o: implicit.h\n");
- fs_.Create("out.o", 1, "");
+ fs_.Create("implicit.h", "");
+ fs_.Create("foo.cc", "");
+ fs_.Create("out.o.d", "out.o: implicit.h\n");
+ fs_.Create("out.o", "");
+ fs_.Tick();
+ fs_.Create("data", "");
Edge* edge = GetNode("out.o")->in_edge();
string err;
@@ -107,9 +110,9 @@
" depfile = $out.d\n"
" command = cat $in > $out\n"
"build ./out.o: catdep ./foo.cc\n"));
- fs_.Create("foo.cc", 1, "");
- fs_.Create("out.o.d", 1, "out.o: foo.cc\n");
- fs_.Create("out.o", 1, "");
+ fs_.Create("foo.cc", "");
+ fs_.Create("out.o.d", "out.o: foo.cc\n");
+ fs_.Create("out.o", "");
Edge* edge = GetNode("out.o")->in_edge();
string err;
@@ -151,9 +154,9 @@
" depfile = $out.d\n"
" command = cat $in > $out\n"
"build ./out.o: catdep ./foo.cc\n"));
- fs_.Create("foo.cc", 1, "");
- fs_.Create("out.o.d", 1, "out.o: bar/../foo.cc\n");
- fs_.Create("out.o", 1, "");
+ fs_.Create("foo.cc", "");
+ fs_.Create("out.o.d", "out.o: bar/../foo.cc\n");
+ fs_.Create("out.o", "");
Edge* edge = GetNode("out.o")->in_edge();
string err;
@@ -170,10 +173,11 @@
" depfile = $out.d\n"
" command = cat $in > $out\n"
"build ./out.o: catdep ./foo.cc\n"));
- fs_.Create("foo.h", 1, "");
- fs_.Create("foo.cc", 1, "");
- fs_.Create("out.o.d", 2, "out.o: foo.h\n");
- fs_.Create("out.o", 2, "");
+ fs_.Create("foo.h", "");
+ fs_.Create("foo.cc", "");
+ fs_.Tick();
+ fs_.Create("out.o.d", "out.o: foo.h\n");
+ fs_.Create("out.o", "");
Edge* edge = GetNode("out.o")->in_edge();
string err;
diff --git a/src/line_printer.cc b/src/line_printer.cc
new file mode 100644
index 0000000..a75eb05
--- /dev/null
+++ b/src/line_printer.cc
@@ -0,0 +1,109 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "line_printer.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/time.h>
+#endif
+
+#include "util.h"
+
+LinePrinter::LinePrinter() : have_blank_line_(true) {
+#ifndef _WIN32
+ const char* term = getenv("TERM");
+ smart_terminal_ = isatty(1) && term && string(term) != "dumb";
+#else
+ // Disable output buffer. It'd be nice to use line buffering but
+ // MSDN says: "For some systems, [_IOLBF] provides line
+ // buffering. However, for Win32, the behavior is the same as _IOFBF
+ // - Full Buffering."
+ setvbuf(stdout, NULL, _IONBF, 0);
+ console_ = GetStdHandle(STD_OUTPUT_HANDLE);
+ CONSOLE_SCREEN_BUFFER_INFO csbi;
+ smart_terminal_ = GetConsoleScreenBufferInfo(console_, &csbi);
+#endif
+}
+
+void LinePrinter::Print(string to_print, LineType type) {
+#ifdef _WIN32
+ CONSOLE_SCREEN_BUFFER_INFO csbi;
+ GetConsoleScreenBufferInfo(console_, &csbi);
+#endif
+
+ if (smart_terminal_) {
+#ifndef _WIN32
+ printf("\r"); // Print over previous line, if any.
+#else
+ csbi.dwCursorPosition.X = 0;
+ SetConsoleCursorPosition(console_, csbi.dwCursorPosition);
+#endif
+ }
+
+ if (smart_terminal_ && type == ELIDE) {
+#ifdef _WIN32
+ // Don't use the full width or console will move to next line.
+ size_t width = static_cast<size_t>(csbi.dwSize.X) - 1;
+ to_print = ElideMiddle(to_print, width);
+ // We don't want to have the cursor spamming back and forth, so
+ // use WriteConsoleOutput instead which updates the contents of
+ // the buffer, but doesn't move the cursor position.
+ GetConsoleScreenBufferInfo(console_, &csbi);
+ COORD buf_size = { csbi.dwSize.X, 1 };
+ COORD zero_zero = { 0, 0 };
+ SMALL_RECT target = {
+ csbi.dwCursorPosition.X, csbi.dwCursorPosition.Y,
+ static_cast<SHORT>(csbi.dwCursorPosition.X + csbi.dwSize.X - 1),
+ csbi.dwCursorPosition.Y
+ };
+ CHAR_INFO* char_data = new CHAR_INFO[csbi.dwSize.X];
+ memset(char_data, 0, sizeof(CHAR_INFO) * csbi.dwSize.X);
+ for (int i = 0; i < csbi.dwSize.X; ++i) {
+ char_data[i].Char.AsciiChar = ' ';
+ char_data[i].Attributes = csbi.wAttributes;
+ }
+ for (size_t i = 0; i < to_print.size(); ++i)
+ char_data[i].Char.AsciiChar = to_print[i];
+ WriteConsoleOutput(console_, char_data, buf_size, zero_zero, &target);
+ delete[] char_data;
+#else
+ // Limit output to width of the terminal if provided so we don't cause
+ // line-wrapping.
+ winsize size;
+ if ((ioctl(0, TIOCGWINSZ, &size) == 0) && size.ws_col) {
+ to_print = ElideMiddle(to_print, size.ws_col);
+ }
+ printf("%s", to_print.c_str());
+ printf("\x1B[K"); // Clear to end of line.
+ fflush(stdout);
+#endif
+
+ have_blank_line_ = false;
+ } else {
+ printf("%s\n", to_print.c_str());
+ }
+}
+
+void LinePrinter::PrintOnNewLine(const string& to_print) {
+ if (!have_blank_line_)
+ printf("\n");
+ printf("%s", to_print.c_str());
+ have_blank_line_ = to_print.empty() || *to_print.rbegin() == '\n';
+}
diff --git a/src/line_printer.h b/src/line_printer.h
new file mode 100644
index 0000000..c292464
--- /dev/null
+++ b/src/line_printer.h
@@ -0,0 +1,53 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_LINE_PRINTER_H_
+#define NINJA_LINE_PRINTER_H_
+
+#include <string>
+using namespace std;
+
+/// Prints lines of text, possibly overprinting previously printed lines
+/// if the terminal supports it.
+class LinePrinter {
+ public:
+ LinePrinter();
+
+ bool is_smart_terminal() const { return smart_terminal_; }
+ void set_smart_terminal(bool smart) { smart_terminal_ = smart; }
+
+ enum LineType {
+ FULL,
+ ELIDE
+ };
+ /// Overprints the current line. If type is ELIDE, elides to_print to fit on
+ /// one line.
+ void Print(string to_print, LineType type);
+
+ /// Prints a string on a new line, not overprinting previous output.
+ void PrintOnNewLine(const string& to_print);
+
+ private:
+ /// Whether we can do fancy terminal control codes.
+ bool smart_terminal_;
+
+ /// Whether the caret is at the beginning of a blank line.
+ bool have_blank_line_;
+
+#ifdef _WIN32
+ void* console_;
+#endif
+};
+
+#endif // NINJA_LINE_PRINTER_H_
diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc
index 14fca73..3593567 100644
--- a/src/manifest_parser.cc
+++ b/src/manifest_parser.cc
@@ -14,10 +14,8 @@
#include "manifest_parser.h"
-#include <assert.h>
-#include <errno.h>
#include <stdio.h>
-#include <string.h>
+#include <vector>
#include "graph.h"
#include "metrics.h"
@@ -329,6 +327,14 @@
edge->implicit_deps_ = implicit;
edge->order_only_deps_ = order_only;
+ // Multiple outputs aren't (yet?) supported with depslog.
+ string deps_type = edge->GetBinding("deps");
+ if (!deps_type.empty() && edge->outputs_.size() > 1) {
+ return lexer_.Error("multiple outputs aren't (yet?) supported by depslog; "
+ "bring this up on the mailing list if it affects you",
+ err);
+ }
+
return true;
}
diff --git a/src/manifest_parser.h b/src/manifest_parser.h
index a08e5af..967dfdd 100644
--- a/src/manifest_parser.h
+++ b/src/manifest_parser.h
@@ -16,13 +16,10 @@
#define NINJA_MANIFEST_PARSER_H_
#include <string>
-#include <vector>
-#include <limits>
using namespace std;
#include "lexer.h"
-#include "string_piece.h"
struct BindingEnv;
struct EvalString;
diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc
index 4ac093f..2638edc 100644
--- a/src/manifest_parser_test.cc
+++ b/src/manifest_parser_test.cc
@@ -14,6 +14,9 @@
#include "manifest_parser.h"
+#include <map>
+#include <vector>
+
#include <gtest/gtest.h>
#include "graph.h"
@@ -71,6 +74,7 @@
"rule cat\n"
" command = a\n"
" depfile = a\n"
+" deps = a\n"
" description = a\n"
" generator = a\n"
" restat = a\n"
@@ -599,6 +603,17 @@
EXPECT_EQ("", err);
}
+TEST_F(ParserTest, MultipleOutputsWithDeps) {
+ State state;
+ ManifestParser parser(&state, NULL);
+ string err;
+ EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n deps = gcc\n"
+ "build a.o b.o: cc c.cc\n",
+ &err));
+ EXPECT_EQ("input:5: multiple outputs aren't (yet?) supported by depslog; "
+ "bring this up on the mailing list if it affects you\n", err);
+}
+
TEST_F(ParserTest, SubNinja) {
files_["test.ninja"] =
"var = inner\n"
@@ -689,7 +704,7 @@
"default $third\n"));
string err;
- std::vector<Node*> nodes = state.DefaultNodes(&err);
+ vector<Node*> nodes = state.DefaultNodes(&err);
EXPECT_EQ("", err);
ASSERT_EQ(3u, nodes.size());
EXPECT_EQ("a", nodes[0]->path());
diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc
index fd9b671..be2a5e0 100644
--- a/src/msvc_helper-win32.cc
+++ b/src/msvc_helper-win32.cc
@@ -39,15 +39,15 @@
return result;
}
+} // anonymous namespace
+
string EscapeForDepfile(const string& path) {
// Depfiles don't escape single \.
return Replace(path, " ", "\\ ");
}
-} // anonymous namespace
-
// static
-string CLWrapper::FilterShowIncludes(const string& line) {
+string CLParser::FilterShowIncludes(const string& line) {
static const char kMagicPrefix[] = "Note: including file: ";
const char* in = line.c_str();
const char* end = in + line.size();
@@ -63,14 +63,14 @@
}
// static
-bool CLWrapper::IsSystemInclude(const string& path) {
+bool CLParser::IsSystemInclude(const string& path) {
// TODO: this is a heuristic, perhaps there's a better way?
return (path.find("program files") != string::npos ||
path.find("microsoft visual studio") != string::npos);
}
// static
-bool CLWrapper::FilterInputFilename(const string& line) {
+bool CLParser::FilterInputFilename(const string& line) {
// TODO: other extensions, like .asm?
return EndsWith(line, ".c") ||
EndsWith(line, ".cc") ||
@@ -78,7 +78,42 @@
EndsWith(line, ".cpp");
}
-int CLWrapper::Run(const string& command, string* extra_output) {
+string CLParser::Parse(const string& output) {
+ string filtered_output;
+
+ // Loop over all lines in the output to process them.
+ size_t start = 0;
+ while (start < output.size()) {
+ size_t end = output.find_first_of("\r\n", start);
+ if (end == string::npos)
+ end = output.size();
+ string line = output.substr(start, end - start);
+
+ string include = FilterShowIncludes(line);
+ if (!include.empty()) {
+ include = IncludesNormalize::Normalize(include, NULL);
+ if (!IsSystemInclude(include))
+ includes_.insert(include);
+ } else if (FilterInputFilename(line)) {
+ // Drop it.
+ // TODO: if we support compiling multiple output files in a single
+ // cl.exe invocation, we should stash the filename.
+ } else {
+ filtered_output.append(line);
+ filtered_output.append("\n");
+ }
+
+ if (end < output.size() && output[end] == '\r')
+ ++end;
+ if (end < output.size() && output[end] == '\n')
+ ++end;
+ start = end;
+ }
+
+ return filtered_output;
+}
+
+int CLWrapper::Run(const string& command, string* output) {
SECURITY_ATTRIBUTES security_attributes = {};
security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES);
security_attributes.bInheritHandle = TRUE;
@@ -118,8 +153,7 @@
Win32Fatal("CloseHandle");
}
- // Read output of the subprocess and parse it.
- string output;
+ // Read all output of the subprocess.
DWORD read_len = 1;
while (read_len) {
char buf[64 << 10];
@@ -128,44 +162,12 @@
GetLastError() != ERROR_BROKEN_PIPE) {
Win32Fatal("ReadFile");
}
- output.append(buf, read_len);
-
- // Loop over all lines in the output and process them.
- for (;;) {
- size_t ofs = output.find_first_of("\r\n");
- if (ofs == string::npos)
- break;
- string line = output.substr(0, ofs);
-
- string include = FilterShowIncludes(line);
- if (!include.empty()) {
- include = IncludesNormalize::Normalize(include, NULL);
- if (!IsSystemInclude(include))
- includes_.insert(include);
- } else if (FilterInputFilename(line)) {
- // Drop it.
- // TODO: if we support compiling multiple output files in a single
- // cl.exe invocation, we should stash the filename.
- } else {
- if (extra_output) {
- extra_output->append(line);
- extra_output->append("\n");
- } else {
- printf("%s\n", line.c_str());
- }
- }
-
- if (ofs < output.size() && output[ofs] == '\r')
- ++ofs;
- if (ofs < output.size() && output[ofs] == '\n')
- ++ofs;
- output = output.substr(ofs);
- }
+ output->append(buf, read_len);
}
+ // Wait for it to exit and grab its exit code.
if (WaitForSingleObject(process_info.hProcess, INFINITE) == WAIT_FAILED)
Win32Fatal("WaitForSingleObject");
-
DWORD exit_code = 0;
if (!GetExitCodeProcess(process_info.hProcess, &exit_code))
Win32Fatal("GetExitCodeProcess");
@@ -178,11 +180,3 @@
return exit_code;
}
-
-vector<string> CLWrapper::GetEscapedResult() {
- vector<string> result;
- for (set<string>::iterator i = includes_.begin(); i != includes_.end(); ++i) {
- result.push_back(EscapeForDepfile(*i));
- }
- return result;
-}
diff --git a/src/msvc_helper.h b/src/msvc_helper.h
index 102201b..32ab606 100644
--- a/src/msvc_helper.h
+++ b/src/msvc_helper.h
@@ -17,23 +17,13 @@
#include <vector>
using namespace std;
+string EscapeForDepfile(const string& path);
+
/// Visual Studio's cl.exe requires some massaging to work with Ninja;
/// for example, it emits include information on stderr in a funny
-/// format when building with /showIncludes. This class wraps a CL
-/// process and parses that output to extract the file list.
-struct CLWrapper {
- CLWrapper() : env_block_(NULL) {}
-
- /// Set the environment block (as suitable for CreateProcess) to be used
- /// by Run().
- void SetEnvBlock(void* env_block) { env_block_ = env_block; }
-
- /// Start a process and parse its output. Returns its exit code.
- /// Any non-parsed output is buffered into \a extra_output if provided,
- /// otherwise it is printed to stdout while the process runs.
- /// Crashes (calls Fatal()) on error.
- int Run(const string& command, string* extra_output=NULL);
-
+/// format when building with /showIncludes. This class parses this
+/// output.
+struct CLParser {
/// Parse a line of cl.exe output and extract /showIncludes info.
/// If a dependency is extracted, returns a nonempty string.
/// Exposed for testing.
@@ -50,10 +40,24 @@
/// Exposed for testing.
static bool FilterInputFilename(const string& line);
- /// Fill a vector with the unique'd headers, escaped for output as a .d
- /// file.
- vector<string> GetEscapedResult();
+ /// Parse the full output of cl, returning the output (if any) that
+ /// should printed.
+ string Parse(const string& output);
+
+ set<string> includes_;
+};
+
+/// Wraps a synchronous execution of a CL subprocess.
+struct CLWrapper {
+ CLWrapper() : env_block_(NULL) {}
+
+ /// Set the environment block (as suitable for CreateProcess) to be used
+ /// by Run().
+ void SetEnvBlock(void* env_block) { env_block_ = env_block; }
+
+ /// Start a process and gather its raw output. Returns its exit code.
+ /// Crashes (calls Fatal()) on error.
+ int Run(const string& command, string* output);
void* env_block_;
- set<string> includes_;
};
diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc
index 0bbe98b..ef91450 100644
--- a/src/msvc_helper_main-win32.cc
+++ b/src/msvc_helper_main-win32.cc
@@ -44,12 +44,13 @@
}
}
-void WriteDepFileOrDie(const char* object_path, CLWrapper* cl) {
+void WriteDepFileOrDie(const char* object_path, const CLParser& parse) {
string depfile_path = string(object_path) + ".d";
FILE* depfile = fopen(depfile_path.c_str(), "w");
if (!depfile) {
unlink(object_path);
- Fatal("opening %s: %s", depfile_path.c_str(), GetLastErrorString().c_str());
+ Fatal("opening %s: %s", depfile_path.c_str(),
+ GetLastErrorString().c_str());
}
if (fprintf(depfile, "%s: ", object_path) < 0) {
unlink(object_path);
@@ -57,9 +58,10 @@
unlink(depfile_path.c_str());
Fatal("writing %s", depfile_path.c_str());
}
- vector<string> headers = cl->GetEscapedResult();
- for (vector<string>::iterator i = headers.begin(); i != headers.end(); ++i) {
- if (fprintf(depfile, "%s\n", i->c_str()) < 0) {
+ const set<string>& headers = parse.includes_;
+ for (set<string>::const_iterator i = headers.begin();
+ i != headers.end(); ++i) {
+ if (fprintf(depfile, "%s\n", EscapeForDepfile(*i).c_str()) < 0) {
unlink(object_path);
fclose(depfile);
unlink(depfile_path.c_str());
@@ -95,11 +97,6 @@
}
}
- if (!output_filename) {
- Usage();
- Fatal("-o required");
- }
-
string env;
if (envfile) {
string err;
@@ -118,9 +115,15 @@
CLWrapper cl;
if (!env.empty())
cl.SetEnvBlock((void*)env.data());
- int exit_code = cl.Run(command);
+ string output;
+ int exit_code = cl.Run(command, &output);
- WriteDepFileOrDie(output_filename, &cl);
+ if (output_filename) {
+ CLParser parser;
+ output = parser.Parse(output);
+ WriteDepFileOrDie(output_filename, parser);
+ }
+ printf("%s\n", output.c_str());
return exit_code;
}
diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc
index 7730425..1e1cbde 100644
--- a/src/msvc_helper_test.cc
+++ b/src/msvc_helper_test.cc
@@ -19,58 +19,86 @@
#include "test.h"
#include "util.h"
-TEST(MSVCHelperTest, ShowIncludes) {
- ASSERT_EQ("", CLWrapper::FilterShowIncludes(""));
+TEST(CLParserTest, ShowIncludes) {
+ ASSERT_EQ("", CLParser::FilterShowIncludes(""));
- ASSERT_EQ("", CLWrapper::FilterShowIncludes("Sample compiler output"));
+ ASSERT_EQ("", CLParser::FilterShowIncludes("Sample compiler output"));
ASSERT_EQ("c:\\Some Files\\foobar.h",
- CLWrapper::FilterShowIncludes("Note: including file: "
- "c:\\Some Files\\foobar.h"));
+ CLParser::FilterShowIncludes("Note: including file: "
+ "c:\\Some Files\\foobar.h"));
ASSERT_EQ("c:\\initspaces.h",
- CLWrapper::FilterShowIncludes("Note: including file: "
- "c:\\initspaces.h"));
+ CLParser::FilterShowIncludes("Note: including file: "
+ "c:\\initspaces.h"));
}
-TEST(MSVCHelperTest, FilterInputFilename) {
- ASSERT_TRUE(CLWrapper::FilterInputFilename("foobar.cc"));
- ASSERT_TRUE(CLWrapper::FilterInputFilename("foo bar.cc"));
- ASSERT_TRUE(CLWrapper::FilterInputFilename("baz.c"));
+TEST(CLParserTest, FilterInputFilename) {
+ ASSERT_TRUE(CLParser::FilterInputFilename("foobar.cc"));
+ ASSERT_TRUE(CLParser::FilterInputFilename("foo bar.cc"));
+ ASSERT_TRUE(CLParser::FilterInputFilename("baz.c"));
- ASSERT_FALSE(CLWrapper::FilterInputFilename(
+ ASSERT_FALSE(CLParser::FilterInputFilename(
"src\\cl_helper.cc(166) : fatal error C1075: end "
"of file found ..."));
}
-TEST(MSVCHelperTest, Run) {
- CLWrapper cl;
- string output;
- cl.Run("cmd /c \"echo foo&& echo Note: including file: foo.h&&echo bar\"",
- &output);
+TEST(CLParserTest, ParseSimple) {
+ CLParser parser;
+ string output = parser.Parse(
+ "foo\r\n"
+ "Note: including file: foo.h\r\n"
+ "bar\r\n");
+
ASSERT_EQ("foo\nbar\n", output);
- ASSERT_EQ(1u, cl.includes_.size());
- ASSERT_EQ("foo.h", *cl.includes_.begin());
+ ASSERT_EQ(1u, parser.includes_.size());
+ ASSERT_EQ("foo.h", *parser.includes_.begin());
}
-TEST(MSVCHelperTest, RunFilenameFilter) {
- CLWrapper cl;
- string output;
- cl.Run("cmd /c \"echo foo.cc&& echo cl: warning\"",
- &output);
+TEST(CLParserTest, ParseFilenameFilter) {
+ CLParser parser;
+ string output = parser.Parse(
+ "foo.cc\r\n"
+ "cl: warning\r\n");
ASSERT_EQ("cl: warning\n", output);
}
-TEST(MSVCHelperTest, RunSystemInclude) {
- CLWrapper cl;
- string output;
- cl.Run("cmd /c \"echo Note: including file: c:\\Program Files\\foo.h&&"
- "echo Note: including file: d:\\Microsoft Visual Studio\\bar.h&&"
- "echo Note: including file: path.h\"",
- &output);
+TEST(CLParserTest, ParseSystemInclude) {
+ CLParser parser;
+ string output = parser.Parse(
+ "Note: including file: c:\\Program Files\\foo.h\r\n"
+ "Note: including file: d:\\Microsoft Visual Studio\\bar.h\r\n"
+ "Note: including file: path.h\r\n");
// We should have dropped the first two includes because they look like
// system headers.
ASSERT_EQ("", output);
- ASSERT_EQ(1u, cl.includes_.size());
- ASSERT_EQ("path.h", *cl.includes_.begin());
+ ASSERT_EQ(1u, parser.includes_.size());
+ ASSERT_EQ("path.h", *parser.includes_.begin());
+}
+
+TEST(CLParserTest, DuplicatedHeader) {
+ CLParser parser;
+ string output = parser.Parse(
+ "Note: including file: foo.h\r\n"
+ "Note: including file: bar.h\r\n"
+ "Note: including file: foo.h\r\n");
+ // We should have dropped one copy of foo.h.
+ ASSERT_EQ("", output);
+ ASSERT_EQ(2u, parser.includes_.size());
+}
+
+TEST(CLParserTest, DuplicatedHeaderPathConverted) {
+ CLParser parser;
+ string output = parser.Parse(
+ "Note: including file: sub/foo.h\r\n"
+ "Note: including file: bar.h\r\n"
+ "Note: including file: sub\\foo.h\r\n");
+ // We should have dropped one copy of foo.h.
+ ASSERT_EQ("", output);
+ ASSERT_EQ(2u, parser.includes_.size());
+}
+
+TEST(CLParserTest, SpacesInFilename) {
+ ASSERT_EQ("sub\\some\\ sdk\\foo.h",
+ EscapeForDepfile("sub\\some sdk\\foo.h"));
}
TEST(MSVCHelperTest, EnvBlock) {
@@ -79,40 +107,5 @@
cl.SetEnvBlock(env_block);
string output;
cl.Run("cmd /c \"echo foo is %foo%", &output);
- ASSERT_EQ("foo is bar\n", output);
-}
-
-TEST(MSVCHelperTest, DuplicatedHeader) {
- CLWrapper cl;
- string output;
- cl.Run("cmd /c \"echo Note: including file: foo.h&&"
- "echo Note: including file: bar.h&&"
- "echo Note: including file: foo.h\"",
- &output);
- // We should have dropped one copy of foo.h.
- ASSERT_EQ("", output);
- ASSERT_EQ(2u, cl.includes_.size());
-}
-
-TEST(MSVCHelperTest, DuplicatedHeaderPathConverted) {
- CLWrapper cl;
- string output;
- cl.Run("cmd /c \"echo Note: including file: sub/foo.h&&"
- "echo Note: including file: bar.h&&"
- "echo Note: including file: sub\\foo.h\"",
- &output);
- // We should have dropped one copy of foo.h.
- ASSERT_EQ("", output);
- ASSERT_EQ(2u, cl.includes_.size());
-}
-
-TEST(MSVCHelperTest, SpacesInFilename) {
- CLWrapper cl;
- string output;
- cl.Run("cmd /c \"echo Note: including file: sub\\some sdk\\foo.h",
- &output);
- ASSERT_EQ("", output);
- vector<string> headers = cl.GetEscapedResult();
- ASSERT_EQ(1u, headers.size());
- ASSERT_EQ("sub\\some\\ sdk\\foo.h", headers[0]);
+ ASSERT_EQ("foo is bar\r\n", output);
}
diff --git a/src/ninja.cc b/src/ninja.cc
index 69646e1..b4797ed 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -17,8 +17,6 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
#ifdef _WIN32
#include "getopt.h"
@@ -32,9 +30,9 @@
#include "browse.h"
#include "build.h"
#include "build_log.h"
+#include "deps_log.h"
#include "clean.h"
#include "disk_interface.h"
-#include "edit_distance.h"
#include "explain.h"
#include "graph.h"
#include "graphviz.h"
@@ -641,20 +639,13 @@
}
}
-bool OpenLog(BuildLog* build_log, Globals* globals,
- DiskInterface* disk_interface) {
- const string build_dir =
- globals->state->bindings_.LookupVariable("builddir");
- const char* kLogPath = ".ninja_log";
- string log_path = kLogPath;
- if (!build_dir.empty()) {
- log_path = build_dir + "/" + kLogPath;
- if (!disk_interface->MakeDirs(log_path) && errno != EEXIST) {
- Error("creating build directory %s: %s",
- build_dir.c_str(), strerror(errno));
- return false;
- }
- }
+/// Open the build log.
+/// @return false on error.
+bool OpenBuildLog(BuildLog* build_log, const string& build_dir,
+ Globals* globals, DiskInterface* disk_interface) {
+ string log_path = ".ninja_log";
+ if (!build_dir.empty())
+ log_path = build_dir + "/" + log_path;
string err;
if (!build_log->Load(log_path, &err)) {
@@ -677,6 +668,36 @@
return true;
}
+/// Open the deps log: load it, then open for writing.
+/// @return false on error.
+bool OpenDepsLog(DepsLog* deps_log, const string& build_dir,
+ Globals* globals, DiskInterface* disk_interface) {
+ string path = ".ninja_deps";
+ if (!build_dir.empty())
+ path = build_dir + "/" + path;
+
+ string err;
+ if (!deps_log->Load(path, globals->state, &err)) {
+ Error("loading deps log %s: %s", path.c_str(), err.c_str());
+ return false;
+ }
+ if (!err.empty()) {
+ // Hack: Load() can return a warning via err by returning true.
+ Warning("%s", err.c_str());
+ err.clear();
+ }
+
+ if (!globals->config->dry_run) {
+ if (!deps_log->OpenForWrite(path, &err)) {
+ Error("opening deps log: %s", err.c_str());
+ return false;
+ }
+ }
+
+ return true;
+}
+
+
/// Dump the output requested by '-d stats'.
void DumpMetrics(Globals* globals) {
g_metrics->Report();
@@ -872,14 +893,30 @@
if (tool && tool->when == Tool::RUN_AFTER_LOAD)
return tool->func(&globals, argc, argv);
- BuildLog build_log;
RealDiskInterface disk_interface;
- if (!OpenLog(&build_log, &globals, &disk_interface))
+
+ // Create the build dir if it doesn't exist.
+ const string build_dir = globals.state->bindings_.LookupVariable("builddir");
+ if (!build_dir.empty() && !config.dry_run) {
+ if (!disk_interface.MakeDirs(build_dir + "/.") &&
+ errno != EEXIST) {
+ Error("creating build directory %s: %s",
+ build_dir.c_str(), strerror(errno));
+ return 1;
+ }
+ }
+
+ BuildLog build_log;
+ if (!OpenBuildLog(&build_log, build_dir, &globals, &disk_interface))
+ return 1;
+
+ DepsLog deps_log;
+ if (!OpenDepsLog(&deps_log, build_dir, &globals, &disk_interface))
return 1;
if (!rebuilt_manifest) { // Don't get caught in an infinite loop by a rebuild
// target that is never up to date.
- Builder manifest_builder(globals.state, config, &build_log,
+ Builder manifest_builder(globals.state, config, &build_log, &deps_log,
&disk_interface);
if (RebuildManifest(&manifest_builder, input_file, &err)) {
rebuilt_manifest = true;
@@ -891,7 +928,8 @@
}
}
- Builder builder(globals.state, config, &build_log, &disk_interface);
+ Builder builder(globals.state, config, &build_log, &deps_log,
+ &disk_interface);
int result = RunBuild(&builder, argc, argv);
if (g_metrics)
DumpMetrics(&globals);
diff --git a/src/ninja_test.cc b/src/ninja_test.cc
new file mode 100644
index 0000000..31754f2
--- /dev/null
+++ b/src/ninja_test.cc
@@ -0,0 +1,89 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdarg.h>
+#include <stdio.h>
+
+#include "gtest/gtest.h"
+#include "line_printer.h"
+
+string StringPrintf(const char* format, ...) {
+ const int N = 1024;
+ char buf[N];
+
+ va_list ap;
+ va_start(ap, format);
+ vsnprintf(buf, N, format, ap);
+ va_end(ap);
+
+ return buf;
+}
+
+/// A test result printer that's less wordy than gtest's default.
+class LaconicPrinter : public testing::EmptyTestEventListener {
+ public:
+ LaconicPrinter() : tests_started_(0), test_count_(0), iteration_(0) {}
+ virtual void OnTestProgramStart(const testing::UnitTest& unit_test) {
+ test_count_ = unit_test.test_to_run_count();
+ }
+
+ virtual void OnTestIterationStart(const testing::UnitTest& test_info,
+ int iteration) {
+ tests_started_ = 0;
+ iteration_ = iteration;
+ }
+
+ virtual void OnTestStart(const testing::TestInfo& test_info) {
+ ++tests_started_;
+ printer_.Print(
+ StringPrintf("[%d/%d%s] %s.%s",
+ tests_started_,
+ test_count_,
+ iteration_ ? StringPrintf(" iter %d", iteration_).c_str()
+ : "",
+ test_info.test_case_name(),
+ test_info.name()),
+ LinePrinter::ELIDE);
+ }
+
+ virtual void OnTestPartResult(
+ const testing::TestPartResult& test_part_result) {
+ if (!test_part_result.failed())
+ return;
+ printer_.PrintOnNewLine(StringPrintf(
+ "*** Failure in %s:%d\n%s\n", test_part_result.file_name(),
+ test_part_result.line_number(), test_part_result.summary()));
+ }
+
+ virtual void OnTestProgramEnd(const testing::UnitTest& unit_test) {
+ printer_.PrintOnNewLine(unit_test.Passed() ? "passed\n" : "failed\n");
+ }
+
+ private:
+ LinePrinter printer_;
+ int tests_started_;
+ int test_count_;
+ int iteration_;
+};
+
+int main(int argc, char **argv) {
+ testing::InitGoogleTest(&argc, argv);
+
+ testing::TestEventListeners& listeners =
+ testing::UnitTest::GetInstance()->listeners();
+ delete listeners.Release(listeners.default_result_printer());
+ listeners.Append(new LaconicPrinter);
+
+ return RUN_ALL_TESTS();
+}
diff --git a/src/state.cc b/src/state.cc
index 9f46fee..9b6160b 100644
--- a/src/state.cc
+++ b/src/state.cc
@@ -154,7 +154,8 @@
edge->outputs_.push_back(node);
if (node->in_edge()) {
Warning("multiple rules generate %s. "
- "build will not be correct; continuing anyway",
+ "builds involving this target will not be correct; "
+ "continuing anyway",
path.AsString().c_str());
}
node->set_in_edge(edge);
@@ -202,10 +203,11 @@
void State::Dump() {
for (Paths::iterator i = paths_.begin(); i != paths_.end(); ++i) {
Node* node = i->second;
- printf("%s %s\n",
+ printf("%s %s [id:%d]\n",
node->path().c_str(),
node->status_known() ? (node->dirty() ? "dirty" : "clean")
- : "unknown");
+ : "unknown",
+ node->id());
}
if (!pools_.empty()) {
printf("resource_pools:\n");
diff --git a/src/state.h b/src/state.h
index 7e3aead..bde75ff 100644
--- a/src/state.h
+++ b/src/state.h
@@ -16,7 +16,6 @@
#define NINJA_STATE_H_
#include <map>
-#include <deque>
#include <set>
#include <string>
#include <vector>
diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc
index 8f1a04e..339edfe 100644
--- a/src/subprocess-posix.cc
+++ b/src/subprocess-posix.cc
@@ -14,8 +14,6 @@
#include "subprocess.h"
-#include <algorithm>
-#include <map>
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
@@ -25,13 +23,6 @@
#include <string.h>
#include <sys/wait.h>
-// Older versions of glibc (like 2.4) won't find this in <poll.h>. glibc
-// 2.4 keeps it in <asm-generic/poll.h>, though attempting to include that
-// will redefine the pollfd structure.
-#ifndef POLLRDHUP
-#define POLLRDHUP 0x2000
-#endif
-
#include "util.h"
Subprocess::Subprocess() : fd_(-1), pid_(-1) {
@@ -49,12 +40,12 @@
if (pipe(output_pipe) < 0)
Fatal("pipe: %s", strerror(errno));
fd_ = output_pipe[0];
-#if !defined(linux)
- // On linux we use ppoll in DoWork(); elsewhere we use pselect and so must
- // avoid overly-large FDs.
+#if !defined(linux) && !defined(__OpenBSD__)
+ // On Linux and OpenBSD, we use ppoll in DoWork(); elsewhere we use pselect
+ // and so must avoid overly-large FDs.
if (fd_ >= static_cast<int>(FD_SETSIZE))
Fatal("pipe: %s", strerror(EMFILE));
-#endif // !linux
+#endif // !linux && !__OpenBSD__
SetCloseOnExec(fd_);
pid_ = fork();
@@ -155,8 +146,6 @@
}
SubprocessSet::SubprocessSet() {
- interrupted_ = false;
-
sigset_t set;
sigemptyset(&set);
sigaddset(&set, SIGINT);
@@ -189,7 +178,7 @@
return subprocess;
}
-#ifdef linux
+#if defined(linux) || defined(__OpenBSD__)
bool SubprocessSet::DoWork() {
vector<pollfd> fds;
nfds_t nfds = 0;
@@ -199,20 +188,19 @@
int fd = (*i)->fd_;
if (fd < 0)
continue;
- pollfd pfd = { fd, POLLIN | POLLPRI | POLLRDHUP, 0 };
+ pollfd pfd = { fd, POLLIN | POLLPRI, 0 };
fds.push_back(pfd);
++nfds;
}
+ interrupted_ = false;
int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_);
if (ret == -1) {
if (errno != EINTR) {
perror("ninja: ppoll");
return false;
}
- bool interrupted = interrupted_;
- interrupted_ = false;
- return interrupted;
+ return interrupted_;
}
nfds_t cur_nfd = 0;
@@ -233,10 +221,10 @@
++i;
}
- return false;
+ return interrupted_;
}
-#else // linux
+#else // linux || __OpenBSD__
bool SubprocessSet::DoWork() {
fd_set set;
int nfds = 0;
@@ -252,15 +240,14 @@
}
}
+ interrupted_ = false;
int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_);
if (ret == -1) {
if (errno != EINTR) {
perror("ninja: pselect");
return false;
}
- bool interrupted = interrupted_;
- interrupted_ = false;
- return interrupted;
+ return interrupted_;
}
for (vector<Subprocess*>::iterator i = running_.begin();
@@ -277,9 +264,9 @@
++i;
}
- return false;
+ return interrupted_;
}
-#endif // linux
+#endif // linux || __OpenBSD__
Subprocess* SubprocessSet::NextFinished() {
if (finished_.empty())
diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc
index c3175da..afd9008 100644
--- a/src/subprocess_test.cc
+++ b/src/subprocess_test.cc
@@ -152,7 +152,7 @@
// OS X's process limit is less than 1025 by default
// (|sysctl kern.maxprocperuid| is 709 on 10.7 and 10.8 and less prior to that).
-#ifdef linux
+#if defined(linux) || defined(__OpenBSD__)
TEST_F(SubprocessTest, SetWithLots) {
// Arbitrary big number; needs to be over 1024 to confirm we're no longer
// hostage to pselect.
@@ -179,7 +179,7 @@
}
ASSERT_EQ(kNumProcs, subprocs_.finished_.size());
}
-#endif // linux
+#endif // linux || __OpenBSD__
// TODO: this test could work on Windows, just not sure how to simply
// read stdin.
diff --git a/src/test.cc b/src/test.cc
index 0138b3a..45a9226 100644
--- a/src/test.cc
+++ b/src/test.cc
@@ -74,7 +74,11 @@
} // anonymous namespace
StateTestWithBuiltinRules::StateTestWithBuiltinRules() {
- AssertParse(&state_,
+ AddCatRule(&state_);
+}
+
+void StateTestWithBuiltinRules::AddCatRule(State* state) {
+ AssertParse(state,
"rule cat\n"
" command = cat $in > $out\n");
}
@@ -94,9 +98,9 @@
ASSERT_EQ(BuildLog::LogEntry::HashCommand(expected), actual);
}
-void VirtualFileSystem::Create(const string& path, int time,
+void VirtualFileSystem::Create(const string& path,
const string& contents) {
- files_[path].mtime = time;
+ files_[path].mtime = now_;
files_[path].contents = contents;
files_created_.insert(path);
}
@@ -109,7 +113,7 @@
}
bool VirtualFileSystem::WriteFile(const string& path, const string& contents) {
- Create(path, 0, contents);
+ Create(path, contents);
return true;
}
diff --git a/src/test.h b/src/test.h
index 37ca1f9..9f29e07 100644
--- a/src/test.h
+++ b/src/test.h
@@ -29,6 +29,12 @@
/// builtin "cat" rule.
struct StateTestWithBuiltinRules : public testing::Test {
StateTestWithBuiltinRules();
+
+ /// Add a "cat" rule to \a state. Used by some tests; it's
+ /// otherwise done by the ctor to state_.
+ void AddCatRule(State* state);
+
+ /// Short way to get a Node by its path from state_.
Node* GetNode(const string& path);
State state_;
@@ -41,8 +47,16 @@
/// of disk state. It also logs file accesses and directory creations
/// so it can be used by tests to verify disk access patterns.
struct VirtualFileSystem : public DiskInterface {
- /// "Create" a file with a given mtime and contents.
- void Create(const string& path, int time, const string& contents);
+ VirtualFileSystem() : now_(1) {}
+
+ /// "Create" a file with contents.
+ void Create(const string& path, const string& contents);
+
+ /// Tick "time" forwards; subsequent file operations will be newer than
+ /// previous ones.
+ int Tick() {
+ return ++now_;
+ }
// DiskInterface
virtual TimeStamp Stat(const string& path);
@@ -63,6 +77,9 @@
FileMap files_;
set<string> files_removed_;
set<string> files_created_;
+
+ /// A simple fake timestamp for file operations.
+ int now_;
};
struct ScopedTempDir {
diff --git a/src/util.cc b/src/util.cc
index 91e8fad..fa72dd2 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -17,6 +17,7 @@
#ifdef _WIN32
#include <windows.h>
#include <io.h>
+#include <share.h>
#endif
#include <errno.h>
@@ -29,6 +30,7 @@
#include <sys/types.h>
#ifndef _WIN32
+#include <unistd.h>
#include <sys/time.h>
#endif
@@ -354,3 +356,21 @@
}
return result;
}
+
+bool Truncate(const string& path, size_t size, string* err) {
+#ifdef _WIN32
+ int fh = _sopen(path.c_str(), _O_RDWR | _O_CREAT, _SH_DENYNO,
+ _S_IREAD | _S_IWRITE);
+ int success = _chsize(fh, size);
+ _close(fh);
+#else
+ int success = truncate(path.c_str(), size);
+#endif
+ // Both truncate() and _chsize() return 0 on success and set errno and return
+ // -1 on failure.
+ if (success < 0) {
+ *err = strerror(errno);
+ return false;
+ }
+ return true;
+}
diff --git a/src/util.h b/src/util.h
index 3c2a297..9740565 100644
--- a/src/util.h
+++ b/src/util.h
@@ -25,8 +25,14 @@
#include <vector>
using namespace std;
+#ifdef _MSC_VER
+#define NORETURN __declspec(noreturn)
+#else
+#define NORETURN __attribute__((noreturn))
+#endif
+
/// Log a fatal message and exit.
-void Fatal(const char* msg, ...);
+NORETURN void Fatal(const char* msg, ...);
/// Log a warning message.
void Warning(const char* msg, ...);
@@ -70,6 +76,9 @@
/// exceeds @a width.
string ElideMiddle(const string& str, size_t width);
+/// Truncates a file to the given size.
+bool Truncate(const string& path, size_t size, string* err);
+
#ifdef _MSC_VER
#define snprintf _snprintf
#define fileno _fileno
@@ -85,7 +94,7 @@
string GetLastErrorString();
/// Calls Fatal() with a function name and GetLastErrorString.
-void Win32Fatal(const char* function);
+NORETURN void Win32Fatal(const char* function);
#endif
#endif // NINJA_UTIL_H_
diff --git a/src/util_test.cc b/src/util_test.cc
index 4776546..1e29053 100644
--- a/src/util_test.cc
+++ b/src/util_test.cc
@@ -101,7 +101,7 @@
}
TEST(CanonicalizePath, UpDir) {
- std::string path, err;
+ string path, err;
path = "../../foo/bar.h";
EXPECT_TRUE(CanonicalizePath(&path, &err));
EXPECT_EQ("../../foo/bar.h", path);
diff --git a/src/version.cc b/src/version.cc
index 45fb040..07167fe 100644
--- a/src/version.cc
+++ b/src/version.cc
@@ -18,7 +18,7 @@
#include "util.h"
-const char* kNinjaVersion = "1.2.0";
+const char* kNinjaVersion = "1.3.0";
void ParseVersion(const string& version, int* major, int* minor) {
size_t end = version.find('.');
@@ -51,7 +51,3 @@
kNinjaVersion, version.c_str());
}
}
-
-
-
-
