Tools/Scripts/run-api-tests - third_party/webkit - Git at Google


            
            #!/usr/bin/perl -w


            # Copyright (C) 2010-2012, 2014-2015 Apple Inc. All rights reserved.

            
            #

            
            # Redistribution and use in source and binary forms, with or without

            
            # modification, are permitted provided that the following conditions

            
            # are met:

            
            # 1. Redistributions of source code must retain the above copyright

            
            #    notice, this list of conditions and the following disclaimer.

            
            # 2. Redistributions in binary form must reproduce the above copyright

            
            #    notice, this list of conditions and the following disclaimer in the

            
            #    documentation and/or other materials provided with the distribution.

            
            #

            
            # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''

            
            # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,

            
            # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

            
            # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS

            
            # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR

            
            # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF

            
            # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS

            
            # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN

            
            # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

            
            # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF

            
            # THE POSSIBILITY OF SUCH DAMAGE.


            use strict;

            
            use warnings;


            use File::Basename;

            
            use FindBin;

            
            use Getopt::Long qw(:config pass_through);

            
            use IPC::Open3;

            
            use lib $FindBin::Bin;

            
            use sigtrap qw(die normal-signals);

            
            use webkitdirs;

            
            use VCSUtils;


            sub buildTestTool();

            
            sub dumpTestsBySuite(\@);

            
            sub listAllTests();

            
            sub runTest($$);

            
            sub runTestsBySuite(\@);

            
            sub prepareEnvironmentForRunningTestTool();

            
            sub archCommandLineArgumentsForRestrictedEnvironmentVariables();

            
            sub testToolPaths();


            # Defined in VCSUtils.

            
            sub possiblyColored($$);


            # Timeout for individual test, in sec

            
            my $timeout = 30;


            my $showHelp = 0;

            
            my $verbose = 0;

            
            my $showLeaks = 0;

            
            my $dumpTests = 0;

            
            my $disableTimeout = 0;

            
            my $build = 1;

            
            my $root;

            
            my $buildDefault = $build ? "build" : "do not build";

            
            my @testsFailed;

            
            my @testsTimedOut;

            
            my $wtfOnly = 0;

            
            my %testToToolMap;


            my $programName = basename($0);

            
            my $usage = <<EOF;

            
            Usage: $programName [options] [suite or test prefixes]

            
              --help                Show this help message

            
              -v|--verbose          Verbose output

            
              -d|--dump-tests       Dump the names of testcases without running them

            
              --[no-]build          Build (or do not build) unit tests prior to running (default: $buildDefault)

            
              --root=               Path to the pre-built root containing TestWebKitAPI

            
              --show-leaks          Show leaks in the output

            
              --no-timeout          Disable test timeouts

            
              --wtf-only            Only build and run TestWTF


            Platform options:

            
              --ios-simulator       Run tests in the iOS Simulator


            @{[ sharedCommandLineOptionsUsage(indent => 2, switchWidth => 21) ]}

            
            Examples


            The following command will run a single test:

            
                $programName WebKit2.AboutBlank


            The following command will run all tests in suites that begin with 'WebKit2':

            
                $programName WebKit2


            EOF


            my $getOptionsResult = GetOptions(

            
                sharedCommandLineOptions(),

            
                'help' => \$showHelp,

            
                'verbose|v' => \$verbose,

            
                'show-leaks' => \$showLeaks,

            
                'no-timeout' => \$disableTimeout,

            
                'dump|d' => \$dumpTests,

            
                'build!' => \$build,

            
                'root=s' => \$root,

            
                'wtf-only' => \$wtfOnly,

            
            );


            if (!$getOptionsResult || $showHelp) {

            
               print STDERR $usage;

            
               exit 1;

            
            }


            setConfiguration();


            setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));


            buildTestTool() if $build && !defined($root);

            
            setPathForRunningWebKitApp(\%ENV);


            my $simulatorDevice;

            
            if (willUseIOSSimulatorSDK()) {

            
                $simulatorDevice = findOrCreateSimulatorForIOSDevice(SIMULATOR_DEVICE_SUFFIX_FOR_WEBKIT_DEVELOPMENT);

            
                relaunchIOSSimulator($simulatorDevice);

            
            }

            
            my @testsToRun = listAllTests();


            @testsToRun = grep { my $test = $_; grep { $test =~ m/^\Q$_\E/ } @ARGV; } @testsToRun if @ARGV;


            if ($dumpTests) {

            
                dumpTestsBySuite(@testsToRun);

            
                exit 0;

            
            }


            END { shutDownIOSSimulatorDevice($simulatorDevice) if $simulatorDevice; }


            exit runTestsBySuite(@testsToRun);


            sub isSupportedPlatform()

            
            {

            
                return isAppleMacWebKit() || isAppleWinWebKit();

            
            }


            sub dumpTestsBySuite(\@)

            
            {

            
                my ($tests) = @_;

            
                print "Dumping test cases\n";

            
                print "------------------\n";

            
                my $lastSuite = "";

            
                for my $suiteAndTest (sort @$tests) {

            
                    my ($suite, $test) = split(/\./, $suiteAndTest);

            
                    if ($lastSuite ne $suite) {

            
                        $lastSuite = $suite;

            
                        print "$suite:\n";

            
                    }

            
                    print "   $test\n";

            
                }

            
                print "------------------\n";

            
            }


            sub runTestsBySuite(\@)

            
            {

            
                my ($tests) = @_;

            
                for my $suiteAndTest (sort @$tests) {

            
                    my ($suite, $test) = split(/\./, $suiteAndTest);

            
                    runTest($suite, $test);

            
                }


                if (@testsFailed) {

            
                    print "\nTests that failed:\n";

            
                    for my $test (@testsFailed) {

            
                        print "  $test\n";

            
                    }

            
                }

            
                if (@testsTimedOut) {

            
                    print "\nTests that timed out:\n";

            
                    for my $test (@testsTimedOut) {

            
                        print "  $test\n";

            
                    }

            
                }


                return @testsFailed > 0 || @testsTimedOut > 0;

            
            }


            sub runTest($$)

            
            {

            
                my ($suite, $testName) = @_;

            
                my $test = $suite . "." . $testName;


                my $gtestArg = "--gtest_filter=" . $test;


                my $result = 0;

            
                my $timedOut = 0;


                die "run-api-tests is not supported on this platform.\n" unless isSupportedPlatform();


                local %ENV = %ENV;

            
                prepareEnvironmentForRunningTestTool();


                local *DEVNULL;

            
                my ($childIn, $childOut, $childErr);

            
                if ($verbose || $showLeaks) {

            
                    $childErr = 0;

            
                } else {

            
                    open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";

            
                    $childErr = ">&DEVNULL";

            
                }


                my $pid;

            
                my @commonArguments = ($testToToolMap{$test}, $gtestArg, @ARGV);

            
                if (willUseIOSSimulatorSDK()) {

            
                    $pid = open3($childIn, $childOut, $childErr, qw(xcrun --sdk iphonesimulator simctl spawn), $simulatorDevice->{UDID}, @commonArguments) or die "Failed to run test: $test.";

            
                } elsif (isAppleMacWebKit() && architecture()) {

            
                    $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), archCommandLineArgumentsForRestrictedEnvironmentVariables(), @commonArguments) or die "Failed to run test: $test.";

            
                } else {

            
                    $pid = open3($childIn, $childOut, $childErr, @commonArguments) or die "Failed to run test: $test.";

            
                }


                eval {

            
                    if ($disableTimeout) {

            
                        waitpid($pid, 0);

            
                    } else {

            
                        local $SIG{ALRM} = sub { die "alarm\n" };

            
                        alarm $timeout;

            
                        waitpid($pid, 0);

            
                        alarm 0;

            
                    }

            
                    $result = $?;

            
                };

            
                if ($@) {

            
                    die unless $@ eq "alarm\n";

            
                    kill SIGTERM, $pid or kill SIGKILL, $pid;

            
                    $timedOut = 1;

            
                }


                my @testOutput = <$childOut>;

            
                @testOutput = grep { !/^LEAK:/ } @testOutput unless $showLeaks;

            
                map { s/\*\*PASS\*\*/possiblyColored("bold green", "PASS")/eg } @testOutput;

            
                map { s/\*\*FAIL\*\*/possiblyColored("bold red", "FAIL")/eg } @testOutput;


                if ($result) {

            
                    push @testsFailed, $test;

            
                    if (!$timedOut && index("@testOutput", $test) == -1) {

            
                        print STDOUT possiblyColored("bold red", "UNEXPECTEDLY EXITED"), " $test\n";

            
                    }

            
                } elsif ($timedOut) {

            
                    push @testsTimedOut, $test;

            
                    print STDOUT possiblyColored("bold yellow", "TIMEOUT"), " $test\n";

            
                }


                print STDOUT @testOutput;


                close($childIn);

            
                close($childOut);

            
                close($childErr) unless ($verbose || $showLeaks);

            
                close(DEVNULL) unless ($verbose || $showLeaks);


                if ($timedOut || $result) {

            
                    return $timedOut || $result;

            
                }


                return 0;

            
            }


            sub listAllTests()

            
            {

            
                my @toolOutput;

            
                my $timedOut;


                die "run-api-tests is not supported on this platform.\n" unless isSupportedPlatform();


                prepareEnvironmentForRunningTestTool();


                local *DEVNULL;

            
                my ($childIn, $childOut, $childErr);

            
                if ($verbose) {

            
                    $childErr = ">&STDERR";

            
                } else {

            
                    open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";

            
                    $childErr = ">&DEVNULL";

            
                }


                my @tests = ();

            
                foreach (testToolPaths()) {

            
                    my $pid;

            
                    my $testTool = $_;

            
                    my @commonArguments = ($testTool, "--gtest_list_tests");

            
                    if (isIOSWebKit()) {

            
                        $pid = open3($childIn, $childOut, $childErr, qw(xcrun --sdk iphonesimulator simctl spawn), $simulatorDevice->{UDID}, @commonArguments) or die "Failed to build list of tests!";

            
                    } elsif (isAppleMacWebKit() && architecture()) {

            
                        $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), archCommandLineArgumentsForRestrictedEnvironmentVariables(), @commonArguments) or die "Failed to build list of tests!";

            
                    } else {

            
                        $pid = open3($childIn, $childOut, $childErr, @commonArguments) or die "Failed to build list of tests!";

            
                    }


                    close($childIn);

            
                    @toolOutput = <$childOut>;

            
                    close($childOut);

            
                    close($childErr);


                    waitpid($pid, 0);

            
                    my $result = $?;


                    if ($result) {

            
                        print STDERR "Failed to build list of tests!--\n";

            
                        exit exitStatus($result);

            
                    }


                    my $suite;

            
                    for my $line (@toolOutput) {

            
                       $line =~ s/[\r\n]*$//;

            
                       if ($line =~ m/\.$/) {

            
                          $suite = $line; # "SuiteName."

            
                       } else {

            
                          # Disabling WebKit2 API test on Windows since we will be disabling WebKit2 on Windows.

            
                          next if (isAppleWinWebKit() && $suite =~ m/WebKit2*/);

            
                          $line =~ s/^\s*//; # "TestName"

            
                          my $fullName = $suite . $line; # "SuiteName.TestName";

            
                          push @tests, $fullName;

            
                          $testToToolMap{$fullName} = $testTool;

            
                        }

            
                    }

            
                }


                close(DEVNULL) unless ($verbose);


                return @tests;

            
            }


            sub buildTestTool()

            
            {

            
                my $originalCwd = getcwd();


                chdirWebKit();


                my $buildTestTool = "build-api-tests";

            
                print STDERR "Running $buildTestTool\n";


                local *DEVNULL;

            
                my ($childIn, $childOut, $childErr);

            
                if ($verbose) {

            
                    # When not quiet, let the child use our stdout/stderr.

            
                    $childOut = ">&STDOUT";

            
                    $childErr = ">&STDERR";

            
                } else {

            
                    open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";

            
                    $childOut = ">&DEVNULL";

            
                    $childErr = ">&DEVNULL";

            
                }


                my @args = argumentsForConfiguration();

            
                if ($wtfOnly) {

            
                    push @args, "--wtf-only";

            
                }


                my $pathToBuildTestTool = File::Spec->catfile("Tools", "Scripts", $buildTestTool);

            
                my $buildProcess = open3($childIn, $childOut, $childErr, "perl", $pathToBuildTestTool, @args) or die "Failed to run " . $buildTestTool;


                close($childIn);

            
                close($childOut);

            
                close($childErr);

            
                close(DEVNULL) unless ($verbose);


                waitpid($buildProcess, 0);

            
                my $buildResult = $?;


                if ($buildResult) {

            
                    print STDERR "Compiling TestWebKitAPI failed!\n";

            
                    exit exitStatus($buildResult);

            
                }


                chdir $originalCwd;

            
            }


            sub prepareEnvironmentForRunningTestTool()

            
            {

            
                return unless isAppleMacWebKit();


                if (willUseIOSSimulatorSDK()) {

            
                    my %simulatorENV;

            
                    {

            
                        local %ENV;

            
                        setupIOSWebKitEnvironment(productDir());

            
                        %simulatorENV = %ENV;

            
                    }

            
                    # Prefix the environment variables with SIMCTL_CHILD_ per `xcrun simctl help launch`.

            
                    foreach my $key (keys %simulatorENV) {

            
                        $ENV{"SIMCTL_CHILD_$key"} = $simulatorENV{$key};

            
                    }

            
                    return;

            
                }

            
                setupMacWebKitEnvironment(productDir());

            
            }


            sub archCommandLineArgumentsForRestrictedEnvironmentVariables()

            
            {

            
                my @arguments = ();

            
                foreach my $key (keys(%ENV)) {

            
                    if ($key =~ /^DYLD_/) {

            
                        push @arguments, "-e", "$key=$ENV{$key}";

            
                    }

            
                }

            
                return @arguments;

            
            }


            sub testToolPaths()

            
            {

            
                if (!isAppleWinWebKit()) {

            
                    my @toolPaths = ();

            
                    if (!$wtfOnly) {

            
                        push @toolPaths, File::Spec->catfile(productDir(), "TestWebKitAPI");

            
                    }

            
                    push @toolPaths, File::Spec->catfile(productDir(), "TestWTF");

            
                    return @toolPaths;

            
                }


                my $binDir = isWin64() ? "bin64" : "bin32";

            
                my $pathWTF = File::Spec->catfile(productDir(), $binDir, "TestWTF");

            
                my $pathWebCore = File::Spec->catfile(productDir(), $binDir, "TestWebCore");

            
                my $pathWebKit = File::Spec->catfile(productDir(), $binDir, "TestWebKit");


                my $suffix;

            
                if (configuration() eq "Debug_All") {

            
                    $suffix = "_debug";

            
                } else {

            
                    $suffix = "";

            
                }

            
                return ("$pathWTF$suffix.exe", "$pathWebCore$suffix.exe", "$pathWebKit$suffix.exe");

            
            }
	#!/usr/bin/perl -w

	# Copyright (C) 2010-2012, 2014-2015 Apple Inc. All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions
	# are met:
	# 1. Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# 2. Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution.
	#
	# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
	# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
	# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
	# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
	# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
	# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
	# THE POSSIBILITY OF SUCH DAMAGE.

	use strict;
	use warnings;

	use File::Basename;
	use FindBin;
	use Getopt::Long qw(:config pass_through);
	use IPC::Open3;
	use lib $FindBin::Bin;
	use sigtrap qw(die normal-signals);
	use webkitdirs;
	use VCSUtils;

	sub buildTestTool();
	sub dumpTestsBySuite(\@);
	sub listAllTests();
	sub runTest($$);
	sub runTestsBySuite(\@);
	sub prepareEnvironmentForRunningTestTool();
	sub archCommandLineArgumentsForRestrictedEnvironmentVariables();
	sub testToolPaths();

	# Defined in VCSUtils.
	sub possiblyColored($$);

	# Timeout for individual test, in sec
	my $timeout = 30;

	my $showHelp = 0;
	my $verbose = 0;
	my $showLeaks = 0;
	my $dumpTests = 0;
	my $disableTimeout = 0;
	my $build = 1;
	my $root;
	my $buildDefault = $build ? "build" : "do not build";
	my @testsFailed;
	my @testsTimedOut;
	my $wtfOnly = 0;
	my %testToToolMap;


	my $programName = basename($0);
	my $usage = <<EOF;
	Usage: $programName [options] [suite or test prefixes]
	--help Show this help message
	-v\|--verbose Verbose output
	-d\|--dump-tests Dump the names of testcases without running them
	--[no-]build Build (or do not build) unit tests prior to running (default: $buildDefault)
	--root= Path to the pre-built root containing TestWebKitAPI
	--show-leaks Show leaks in the output
	--no-timeout Disable test timeouts
	--wtf-only Only build and run TestWTF

	Platform options:
	--ios-simulator Run tests in the iOS Simulator

	@{[ sharedCommandLineOptionsUsage(indent => 2, switchWidth => 21) ]}
	Examples

	The following command will run a single test:
	$programName WebKit2.AboutBlank

	The following command will run all tests in suites that begin with 'WebKit2':
	$programName WebKit2

	EOF

	my $getOptionsResult = GetOptions(
	sharedCommandLineOptions(),
	'help' => \$showHelp,
	'verbose\|v' => \$verbose,
	'show-leaks' => \$showLeaks,
	'no-timeout' => \$disableTimeout,
	'dump\|d' => \$dumpTests,
	'build!' => \$build,
	'root=s' => \$root,
	'wtf-only' => \$wtfOnly,
	);

	if (!$getOptionsResult \|\| $showHelp) {
	print STDERR $usage;
	exit 1;
	}

	setConfiguration();

	setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));

	buildTestTool() if $build && !defined($root);
	setPathForRunningWebKitApp(\%ENV);

	my $simulatorDevice;
	if (willUseIOSSimulatorSDK()) {
	$simulatorDevice = findOrCreateSimulatorForIOSDevice(SIMULATOR_DEVICE_SUFFIX_FOR_WEBKIT_DEVELOPMENT);
	relaunchIOSSimulator($simulatorDevice);
	}
	my @testsToRun = listAllTests();

	@testsToRun = grep { my $test = $_; grep { $test =~ m/^\Q$_\E/ } @ARGV; } @testsToRun if @ARGV;

	if ($dumpTests) {
	dumpTestsBySuite(@testsToRun);
	exit 0;
	}

	END { shutDownIOSSimulatorDevice($simulatorDevice) if $simulatorDevice; }

	exit runTestsBySuite(@testsToRun);

	sub isSupportedPlatform()
	{
	return isAppleMacWebKit() \|\| isAppleWinWebKit();
	}

	sub dumpTestsBySuite(\@)
	{
	my ($tests) = @_;
	print "Dumping test cases\n";
	print "------------------\n";
	my $lastSuite = "";
	for my $suiteAndTest (sort @$tests) {
	my ($suite, $test) = split(/\./, $suiteAndTest);
	if ($lastSuite ne $suite) {
	$lastSuite = $suite;
	print "$suite:\n";
	}
	print " $test\n";
	}
	print "------------------\n";
	}

	sub runTestsBySuite(\@)
	{
	my ($tests) = @_;
	for my $suiteAndTest (sort @$tests) {
	my ($suite, $test) = split(/\./, $suiteAndTest);
	runTest($suite, $test);
	}

	if (@testsFailed) {
	print "\nTests that failed:\n";
	for my $test (@testsFailed) {
	print " $test\n";
	}
	}
	if (@testsTimedOut) {
	print "\nTests that timed out:\n";
	for my $test (@testsTimedOut) {
	print " $test\n";
	}
	}

	return @testsFailed > 0 \|\| @testsTimedOut > 0;
	}

	sub runTest($$)
	{
	my ($suite, $testName) = @_;
	my $test = $suite . "." . $testName;

	my $gtestArg = "--gtest_filter=" . $test;

	my $result = 0;
	my $timedOut = 0;

	die "run-api-tests is not supported on this platform.\n" unless isSupportedPlatform();

	local %ENV = %ENV;
	prepareEnvironmentForRunningTestTool();

	local *DEVNULL;
	my ($childIn, $childOut, $childErr);
	if ($verbose \|\| $showLeaks) {
	$childErr = 0;
	} else {
	open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
	$childErr = ">&DEVNULL";
	}

	my $pid;
	my @commonArguments = ($testToToolMap{$test}, $gtestArg, @ARGV);
	if (willUseIOSSimulatorSDK()) {
	$pid = open3($childIn, $childOut, $childErr, qw(xcrun --sdk iphonesimulator simctl spawn), $simulatorDevice->{UDID}, @commonArguments) or die "Failed to run test: $test.";
	} elsif (isAppleMacWebKit() && architecture()) {
	$pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), archCommandLineArgumentsForRestrictedEnvironmentVariables(), @commonArguments) or die "Failed to run test: $test.";
	} else {
	$pid = open3($childIn, $childOut, $childErr, @commonArguments) or die "Failed to run test: $test.";
	}

	eval {
	if ($disableTimeout) {
	waitpid($pid, 0);
	} else {
	local $SIG{ALRM} = sub { die "alarm\n" };
	alarm $timeout;
	waitpid($pid, 0);
	alarm 0;
	}
	$result = $?;
	};
	if ($@) {
	die unless $@ eq "alarm\n";
	kill SIGTERM, $pid or kill SIGKILL, $pid;
	$timedOut = 1;
	}

	my @testOutput = <$childOut>;
	@testOutput = grep { !/^LEAK:/ } @testOutput unless $showLeaks;
	map { s/\\PASS\\/possiblyColored("bold green", "PASS")/eg } @testOutput;
	map { s/\\FAIL\\/possiblyColored("bold red", "FAIL")/eg } @testOutput;

	if ($result) {
	push @testsFailed, $test;
	if (!$timedOut && index("@testOutput", $test) == -1) {
	print STDOUT possiblyColored("bold red", "UNEXPECTEDLY EXITED"), " $test\n";
	}
	} elsif ($timedOut) {
	push @testsTimedOut, $test;
	print STDOUT possiblyColored("bold yellow", "TIMEOUT"), " $test\n";
	}

	print STDOUT @testOutput;

	close($childIn);
	close($childOut);
	close($childErr) unless ($verbose \|\| $showLeaks);
	close(DEVNULL) unless ($verbose \|\| $showLeaks);

	if ($timedOut \|\| $result) {
	return $timedOut \|\| $result;
	}

	return 0;
	}

	sub listAllTests()
	{
	my @toolOutput;
	my $timedOut;

	die "run-api-tests is not supported on this platform.\n" unless isSupportedPlatform();

	prepareEnvironmentForRunningTestTool();

	local *DEVNULL;
	my ($childIn, $childOut, $childErr);
	if ($verbose) {
	$childErr = ">&STDERR";
	} else {
	open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
	$childErr = ">&DEVNULL";
	}

	my @tests = ();
	foreach (testToolPaths()) {
	my $pid;
	my $testTool = $_;
	my @commonArguments = ($testTool, "--gtest_list_tests");
	if (isIOSWebKit()) {
	$pid = open3($childIn, $childOut, $childErr, qw(xcrun --sdk iphonesimulator simctl spawn), $simulatorDevice->{UDID}, @commonArguments) or die "Failed to build list of tests!";
	} elsif (isAppleMacWebKit() && architecture()) {
	$pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), archCommandLineArgumentsForRestrictedEnvironmentVariables(), @commonArguments) or die "Failed to build list of tests!";
	} else {
	$pid = open3($childIn, $childOut, $childErr, @commonArguments) or die "Failed to build list of tests!";
	}

	close($childIn);
	@toolOutput = <$childOut>;
	close($childOut);
	close($childErr);

	waitpid($pid, 0);
	my $result = $?;

	if ($result) {
	print STDERR "Failed to build list of tests!--\n";
	exit exitStatus($result);
	}

	my $suite;
	for my $line (@toolOutput) {
	$line =~ s/[\r\n]*$//;
	if ($line =~ m/\.$/) {
	$suite = $line; # "SuiteName."
	} else {
	# Disabling WebKit2 API test on Windows since we will be disabling WebKit2 on Windows.
	next if (isAppleWinWebKit() && $suite =~ m/WebKit2*/);
	$line =~ s/^\s*//; # "TestName"
	my $fullName = $suite . $line; # "SuiteName.TestName";
	push @tests, $fullName;
	$testToToolMap{$fullName} = $testTool;
	}
	}
	}

	close(DEVNULL) unless ($verbose);

	return @tests;
	}

	sub buildTestTool()
	{
	my $originalCwd = getcwd();

	chdirWebKit();

	my $buildTestTool = "build-api-tests";
	print STDERR "Running $buildTestTool\n";

	local *DEVNULL;
	my ($childIn, $childOut, $childErr);
	if ($verbose) {
	# When not quiet, let the child use our stdout/stderr.
	$childOut = ">&STDOUT";
	$childErr = ">&STDERR";
	} else {
	open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
	$childOut = ">&DEVNULL";
	$childErr = ">&DEVNULL";
	}

	my @args = argumentsForConfiguration();
	if ($wtfOnly) {
	push @args, "--wtf-only";
	}

	my $pathToBuildTestTool = File::Spec->catfile("Tools", "Scripts", $buildTestTool);
	my $buildProcess = open3($childIn, $childOut, $childErr, "perl", $pathToBuildTestTool, @args) or die "Failed to run " . $buildTestTool;

	close($childIn);
	close($childOut);
	close($childErr);
	close(DEVNULL) unless ($verbose);

	waitpid($buildProcess, 0);
	my $buildResult = $?;

	if ($buildResult) {
	print STDERR "Compiling TestWebKitAPI failed!\n";
	exit exitStatus($buildResult);
	}

	chdir $originalCwd;
	}

	sub prepareEnvironmentForRunningTestTool()
	{
	return unless isAppleMacWebKit();

	if (willUseIOSSimulatorSDK()) {
	my %simulatorENV;
	{
	local %ENV;
	setupIOSWebKitEnvironment(productDir());
	%simulatorENV = %ENV;
	}
	# Prefix the environment variables with SIMCTL_CHILD_ per `xcrun simctl help launch`.
	foreach my $key (keys %simulatorENV) {
	$ENV{"SIMCTL_CHILD_$key"} = $simulatorENV{$key};
	}
	return;
	}
	setupMacWebKitEnvironment(productDir());
	}

	sub archCommandLineArgumentsForRestrictedEnvironmentVariables()
	{
	my @arguments = ();
	foreach my $key (keys(%ENV)) {
	if ($key =~ /^DYLD_/) {
	push @arguments, "-e", "$key=$ENV{$key}";
	}
	}
	return @arguments;
	}

	sub testToolPaths()
	{
	if (!isAppleWinWebKit()) {
	my @toolPaths = ();
	if (!$wtfOnly) {
	push @toolPaths, File::Spec->catfile(productDir(), "TestWebKitAPI");
	}
	push @toolPaths, File::Spec->catfile(productDir(), "TestWTF");
	return @toolPaths;
	}

	my $binDir = isWin64() ? "bin64" : "bin32";
	my $pathWTF = File::Spec->catfile(productDir(), $binDir, "TestWTF");
	my $pathWebCore = File::Spec->catfile(productDir(), $binDir, "TestWebCore");
	my $pathWebKit = File::Spec->catfile(productDir(), $binDir, "TestWebKit");

	my $suffix;
	if (configuration() eq "Debug_All") {
	$suffix = "_debug";
	} else {
	$suffix = "";
	}
	return ("$pathWTF$suffix.exe", "$pathWebCore$suffix.exe", "$pathWebKit$suffix.exe");
	}