Tools/Scripts/run-javascriptcore-tests

#!/usr/bin/perl -w

# Copyright (C) 2005, 2013-2016 Apple Inc.  All rights reserved.
# Copyright (C) 2007 Eric Seidel <eric@webkit.org>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
# 3.  Neither the name of Apple Inc. ("Apple") nor the names of
#     its contributors may be used to endorse or promote products derived
#     from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Script to run the WebKit Open Source Project JavaScriptCore tests (adapted from Mozilla),
# as well as other tests: testapi on Mac and LayoutTests/js.

use strict;
use File::Spec;
use FindBin;
use Getopt::Long qw(:config pass_through);
use JSON::PP;
use lib $FindBin::Bin;
use List::Util qw(min max);
use POSIX;
use webkitdirs;

# determine configuration
setConfiguration();
my $configuration = configuration();

# These variables are intentionally left undefined.
my $root;
my $showHelp;
my @extraTests = ();
my $jsDriverArgs;
my $childProcesses;
my $shellRunner;
my $makeRunner;
my $memoryLimited;

my $buildJSC = 1;

my $runTestAPI = isAppleMacWebKit() || isAppleWinWebKit() || isWinCairo();

my $runJSCStress = 1;
my $runJITStressTests = 1;
my $runMozillaTests = 1;
my $runQuickMode = 0;
my $envVars = "";
my $gmallocPath = undef;
my $gmallocDefaultPath = "/usr/lib/libgmalloc.dylib";

my $createTarball = 0;
my $remoteHost = 0;
my $failFast = 1;
my %jsonData = ();
my $remoteConfigFile;
my $jsonFileName;

if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI}) {
    if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI} eq "true") {
        $runTestAPI = 1;
    } elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI} eq "false") {
        $runTestAPI = 0;
    } else {
        print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_TESTAPI environment variable: '"
            . $ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI} . "'. Should be set to 'true' or 'false'.\n";
    }
}

if ($ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD}) {
    if ($ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD} eq "true") {
        $buildJSC = 1;
    } elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD} eq "false") {
        $buildJSC = 0;
    } else {
        print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_BUILD environment variable: '"
            . $ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD} . "'. Should be set to 'true' or 'false'.\n";
    }
}

if ($ENV{RUN_JAVASCRIPTCORE_TESTS_EXTRA_TESTS}) {
    push @extraTests, $ENV{RUN_JAVASCRIPTCORE_TESTS_EXTRA_TESTS};
}

my $programName = basename($0);
my $buildJSCDefault = $buildJSC ? "will check" : "will not check";
my $testapiDefault = $runTestAPI ? "will run" : "will not run";
my $jscStressDefault = $runJSCStress ? "will run" : " will not run";
my $jitStressTestsDefault = $runJITStressTests ? "will run" : " will not run";
my $mozillaTestsDefault = $runMozillaTests ? "will run" : " will not run";
my $quickModeDefault = $runQuickMode ? "some" : "all";
my $failFastDefault = $failFast ? "fail fast" : "don't fail fast";
my $filter;
my $usage = <<EOF;
Usage: $programName [options] [options to pass to build system]
  --help                        Show this help message
  --root=                       Path to pre-built root containing jsc
  --[no-]ftl-jit                Turn the FTL JIT on or off
  --[no-]build                  Check (or don't check) to see if the jsc build is up-to-date (default: $buildJSCDefault)
  --[no-]testapi                Run (or don't run) testapi (default: $testapiDefault)
  --[no-]jsc-stress             Run (or don't run) the JSC stress tests (default: $jscStressDefault)
  --[no-]jit-stress-tests       Run (or don't run) the JIT stress tests (default: $jitStressTestsDefault)
  --[no-]mozilla-tests          Run (or don't run) the Mozilla tests (default: $mozillaTestsDefault)
  --[no-]quick                  Run some (or all) of the regular testing modes (default: $quickModeDefault)
                                If the runner only runs some it will run the default and no-cjit-validate modes.
                                Note, this will not change the behavior of tests that specify their own modes.

  --[no-]fail-fast              Stop this script when a test family reports an error or failure (default: $failFastDefault)
  --json-output=                Create a file at specified path, listing failed stress tests in JSON format.
  --tarball                     Create a tarball of the bundle produced by running the JSC stress tests.
  --remote=                     Run the JSC stress tests on the specified remote host. Implies --tarball.
  --remote-config-file=         Same as remote, but read config from JSON file.
  --extra-tests=                Path to a file containing extra tests
  --child-processes=            Specify the number of child processes.
  --shell-runner                Uses the shell-based test runner instead of the default make-based runner.
                                In general the shell runner is slower than the make runner.
  --make-runner                 Uses the faster make-based runner.

  --memory-limited              Indicate that we are targeting the test for a memory limited device.
                                Skip tests tagged with //\@largeHeap

  --filter                      Only run tests whose name matches the given regular expression.
  --env-vars                    Pass a list of environment variables to set before running tests.
                                Each environment variable should be separated by a space.
                                e.g. \"foo=bar x=y\" (no quotes).
  --gmalloc:                    Run tests with Guard Malloc enabled (if no path is given: $gmallocDefaultPath is used)

Environment Variables:
  - set RUN_JAVASCRIPTCORE_TESTS_TESTAPI to "true" or "false" (no quotes) to determine if we run the testapi tests.
  - set RUN_JAVASCRIPTCORE_TESTS_BUILD to "true" or "false" (no quotes) to set the should-we-build-before-running-tests setting.
  - set RUN_JAVASCRIPTCORE_TESTS_EXTRA_TESTS to the path of a yaml file or a directory of JS files to be run as part of run-javascriptcore-tests.
EOF

GetOptions(
    'root=s' => \$root,
    'extra-tests=s' => \@extraTests,
    'build!' => \$buildJSC,
    'testapi!' => \$runTestAPI,
    'jsc-stress!' => \$runJSCStress,
    'jit-stress-tests!' => \$runJITStressTests,
    'mozilla-tests!' => \$runMozillaTests,
    'quick!' => \$runQuickMode,
    'fail-fast!' => \$failFast,
    'json-output=s' => \$jsonFileName,
    'tarball!' => \$createTarball,
    'remote=s' => \$remoteHost,
    'remote-config-file=s' => \$remoteConfigFile,
    'child-processes=s' => \$childProcesses,
    'shell-runner' => \$shellRunner,
    'make-runner' => \$makeRunner,
    'memory-limited' => \$memoryLimited,
    'filter=s' => \$filter,
    'help' => \$showHelp,
    'env-vars=s' => \$envVars,
    'gmalloc:s' => \$gmallocPath,
);

# Assume any arguments left over from GetOptions are assumed to be build arguments
my @buildArgs = @ARGV;

if ($showHelp) {
   print STDERR $usage;
   exit 1;
}

setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));

if (defined($jsonFileName)) {
    $jsonFileName = File::Spec->rel2abs($jsonFileName);
}

if (!defined($root) && $buildJSC) {
    chdirWebKit();

    push(@buildArgs, argumentsForConfiguration());

    print "Running: build-jsc " . join(" ", @buildArgs) . "\n";
    my $buildResult = system "perl", File::Spec->catfile("Tools", "Scripts", "build-jsc"), @buildArgs;
    if ($buildResult) {
        print STDERR "Compiling jsc failed!\n";
        exit exitStatus($buildResult);
    }
}

if (defined($gmallocPath)) {
    if ($gmallocPath eq "") {
        $envVars .= " DYLD_INSERT_LIBRARIES=" . $gmallocDefaultPath;
    } else {
        $envVars .= " DYLD_INSERT_LIBRARIES=" . $gmallocPath;
    }
}

my $productDir = jscProductDir();
$ENV{DYLD_FRAMEWORK_PATH} = $productDir;
$ENV{JSCTEST_timeout} = 120 unless $ENV{JSCTEST_timeout}; # Set a 120 second timeout on all jsc tests (if environment variable not defined already).
$ENV{TZ}="US/Pacific"; # Some tests fail if the time zone is not set to US/Pacific (<https://webkit.org/b/136363>)
setPathForRunningWebKitApp(\%ENV) if isCygwin();

sub testapiPath($)
{
    my ($productDir) = @_;
    my $jscName = "testapi";
    $jscName .= "_debug" if configuration() eq "Debug_All";
    return File::Spec->catfile($productDir, $jscName);
}

#run api tests
if ($runTestAPI) {
    chdirWebKit();
    chdir($productDir) or die "Failed to switch directory to '$productDir'\n";
    my @command = (testapiPath($productDir));
    unshift @command, ("xcrun", "-sdk", xcodeSDK(), "sim") if willUseIOSSimulatorSDK();

    if ($envVars ne "") {
        foreach my $var (split(/\s+/, $envVars)) {
            if ($var =~ /([^=]*)=(.*)/) {
                $ENV{$1} = $2;
            }
        }
    }

    # Use an "indirect object" so that system() won't get confused if the path
    # contains spaces (see perldoc -f exec).
    my $testapiResult = system { $command[0] } @command;
    my $exitStatus = exitStatus($testapiResult);
    print "testAPI completed with rc=$testapiResult ($exitStatus)\n";

    if (defined($jsonFileName)) {
        my $apiStatus = ($exitStatus == 0)? JSON::PP::true: JSON::PP::false;
        $jsonData{'allApiTestsPassed'} = $apiStatus;
    }

    if ($testapiResult && $failFast) {
        writeJsonDataIfApplicable();
        exit exitStatus($testapiResult);
    }
}

# Find JavaScriptCore directory
chdirWebKit();

runJSCStressTests();

sub runJSCStressTests
{
    my $jscStressResultsDir = $productDir . "/jsc-stress-results";

    my @testList;
    if ($runJSCStress) {
        @testList = (
            "PerformanceTests/SunSpider/tests/sunspider-1.0",
            "PerformanceTests/JetStream/cdjs/cdjs-tests.yaml",
            "PerformanceTests/ES6SampleBench/Air/airjs-tests.yaml",
            "PerformanceTests/ES6SampleBench/Basic/basic-tests.yaml",
            "JSTests/executableAllocationFuzz.yaml",
            "JSTests/exceptionFuzz.yaml",
            "PerformanceTests/SunSpider/no-architecture-specific-optimizations.yaml",
            "PerformanceTests/SunSpider/shadow-chicken.yaml",
            "PerformanceTests/SunSpider/tests/v8-v6",
            "JSTests/stress",
            "JSTests/microbenchmarks",
            "PerformanceTests/SunSpider/profiler-test.yaml",
            "LayoutTests/jsc-layout-tests.yaml",
            "JSTests/typeProfiler.yaml",
            "JSTests/controlFlowProfiler.yaml",
            "JSTests/es6.yaml",
            "JSTests/modules.yaml",
            "JSTests/ChakraCore.yaml");
    }

    if ($runMozillaTests) {
        push(@testList, "JSTests/mozilla/mozilla-tests.yaml");
    }

    # Set LANG environment variable so the stress tests will work with newer ruby (<rdar://problem/15010705>)
    $ENV{LANG}="en_US.UTF-8";
    my @jscStressDriverCmd = (
        "/usr/bin/env", "ruby", "Tools/Scripts/run-jsc-stress-tests",
        "-j", jscPath($productDir), "-o", $jscStressResultsDir);

    push(@jscStressDriverCmd, @testList);

    if (isWindows() && !isCygwin()) {
        shift @jscStressDriverCmd; # Remove /usr/bin/env
    }

    if (configuration() eq "Debug") {
        push(@jscStressDriverCmd, "--debug");
    }

    if ($envVars ne "") {
            push(@jscStressDriverCmd, "--env-vars");
            push(@jscStressDriverCmd, $envVars);
    }

    if ($runQuickMode) {
        push(@jscStressDriverCmd, "--quick");
    }

    if (!$runJITStressTests) {
        push(@jscStressDriverCmd, "--no-jit");
    }
    if ($createTarball) {
        push(@jscStressDriverCmd, "--tarball");
    }

    if ($remoteHost) {
        push(@jscStressDriverCmd, "--remote");
        push(@jscStressDriverCmd, $remoteHost);
    }

    if ($remoteConfigFile) {
        push(@jscStressDriverCmd, "--remote-config-file");
        push(@jscStressDriverCmd, $remoteConfigFile);
    }

    if ($childProcesses) {
        push(@jscStressDriverCmd, "--child-processes");
        push(@jscStressDriverCmd, $childProcesses);
    }

    if ($shellRunner) {
        push(@jscStressDriverCmd, "--shell-runner");
    }

    if ($makeRunner) {
        push(@jscStressDriverCmd, "--make-runner");
    }

    if ($memoryLimited) {
        push(@jscStressDriverCmd, "--memory-limited");
    }

    if ($filter) {
        push(@jscStressDriverCmd, "--filter");
        push(@jscStressDriverCmd, $filter);
    }

    # End option processing, the rest of the arguments are tests
    push(@jscStressDriverCmd, "--");

    for my $testSuite (@extraTests) {
        push(@jscStressDriverCmd, $testSuite);
    }
    if (defined($ENV{"EXTRA_JSC_TESTS"})) {
        push(@jscStressDriverCmd, $ENV{"EXTRA_JSC_TESTS"});
    }
    print "Running: " . join(" ", @jscStressDriverCmd) . "\n";
    my $result = system(@jscStressDriverCmd);
    exit exitStatus($result) if $result;

    my @jscStressFailList = readAllLines($jscStressResultsDir . "/failed");
    @jscStressFailList = sort @jscStressFailList;
    my $numJSCStressFailures = @jscStressFailList;

    if ($numJSCStressFailures) {
        print "\n** The following JSC stress test failures have been introduced:\n";
        foreach my $testFailure (@jscStressFailList) {
            print "\t$testFailure\n";
        }
    }
    print "\n";

    print "Results for JSC stress tests:\n";
    printThingsFound($numJSCStressFailures, "failure", "failures", "found");
    print "    OK.\n" if $numJSCStressFailures == 0;

    print "\n";

    if (defined($jsonFileName)) {
        $jsonData{'stressTestFailures'} = \@jscStressFailList;
    }

    writeJsonDataIfApplicable();
    exit(1) if $numJSCStressFailures;
}

sub readAllLines
{
    my ($filename) = @_;
    my @array = ();
    eval {
        open FILE, $filename or die;
        while (<FILE>) {
            chomp;
            push @array, $_;
        }
        close FILE;
    };
    return @array;
}

sub printThingsFound
{
    my ($number, $label, $pluralLabel, $verb) = @_;
    print "    $number ";
    if ($number == 1) {
        print $label;
    } else {
        print $pluralLabel;
    }
    print " $verb.\n";
}

sub writeJsonDataIfApplicable
{
    if (defined($jsonFileName)) {
        open(my $fileHandler, ">", $jsonFileName) or die;
        print $fileHandler "${\encode_json(\%jsonData)}\n";
        close($fileHandler);
    }
}