blob: 3b5984b4c077cb6ddbcde67b3b3624a76e621d99 [file] [log] [blame]
#!/usr/bin/env ruby
# Copyright (C) 2013-2016 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
require 'fileutils'
require 'getoptlong'
require 'pathname'
require 'rbconfig'
require 'uri'
require 'yaml'
module URI
class SSH < Generic
DEFAULT_PORT = 22
end
@@schemes['SSH'] = SSH
end
class String
def scrub
encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
end
end
THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
begin
require 'shellwords'
rescue Exception => e
$stderr.puts "Warning: did not find shellwords, not running any tests."
exit 0
end
$canRunDisplayProfilerOutput = false
begin
require 'rubygems'
require 'json'
require 'highline'
$canRunDisplayProfilerOutput = true
rescue Exception => e
$stderr.puts "Warning: did not find json or highline; some features will be disabled."
$stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
$stderr.puts "Error: #{e.inspect}"
end
def printCommandArray(*cmd)
begin
commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
rescue
commandArray = cmd.join(' ')
end
$stderr.puts ">> #{commandArray}"
end
def mysys(*cmd)
printCommandArray(*cmd) if $verbosity >= 1
raise "Command failed: #{$?.inspect}" unless system(*cmd)
end
def escapeAll(array)
array.map {
| v |
raise "Detected a non-string in #{inspect}" unless v.is_a? String
Shellwords.shellescape(v)
}.join(' ')
end
$jscPath = nil
$doNotMessWithVMPath = false
$jitTests = true
$memoryLimited = false
$outputDir = Pathname.new("results")
$verbosity = 0
$bundle = nil
$tarball = false
$tarFileName = "payload.tar.gz"
$copyVM = false
$testRunnerType = nil
$remoteUser = nil
$remoteHost = nil
$remotePort = nil
$remoteDirectory = nil
$architecture = nil
$hostOS = nil
$filter = nil
$envVars = []
$quickMode = false
def usage
puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
puts
puts "--jsc (-j) Path to JavaScriptCore build product. This option is required."
puts "--no-copy Do not copy the JavaScriptCore build product before testing."
puts " --jsc specifies an already present JavaScriptCore to test."
puts "--memory-limited Indicate that we are targeting the test for a memory limited device."
puts " Skip tests tagged with //@largeHeap"
puts "--no-jit Do not run JIT specific tests."
puts "--output-dir (-o) Path where to put results. Default is #{$outputDir}."
puts "--verbose (-v) Print more things while running."
puts "--run-bundle Runs a bundle previously created by run-jsc-stress-tests."
puts "--tarball [fileName] Creates a tarball of the final bundle. Use name if supplied for tar file."
puts "--arch Specify architecture instead of determining from JavaScriptCore build."
puts " e.g. x86, x86_64, arm."
puts "--os Specify os instead of determining from JavaScriptCore build."
puts " e.g. darwin, linux & windows."
puts "--shell-runner Uses the shell-based test runner instead of the default make-based runner."
puts " In general the shell runner is slower than the make runner."
puts "--make-runner Uses the faster make-based runner."
puts "--remote Specify a remote host on which to run tests from command line argument."
puts "--remote-config-file Specify a remote host on which to run tests from JSON file."
puts "--child-processes (-c) Specify the number of child processes."
puts "--filter Only run tests whose name matches the given regular expression."
puts "--help (-h) Print this message."
puts "--env-vars Add a list of environment variables to set before running jsc."
puts " Each environment variable should be separated by a space."
puts " e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
puts " it will override the default value."
puts "--quick (-q) Only run with the default and no-cjit-validate modes."
exit 1
end
jscArg = nil
ifJSCArgIsntProvidedAreWeReleaseBuild = true
GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
['--no-copy', GetoptLong::NO_ARGUMENT],
['--memory-limited', GetoptLong::NO_ARGUMENT],
['--no-jit', GetoptLong::NO_ARGUMENT],
['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
['--force-vm-copy', GetoptLong::NO_ARGUMENT],
['--arch', GetoptLong::REQUIRED_ARGUMENT],
['--os', GetoptLong::REQUIRED_ARGUMENT],
['--shell-runner', GetoptLong::NO_ARGUMENT],
['--make-runner', GetoptLong::NO_ARGUMENT],
['--remote', GetoptLong::REQUIRED_ARGUMENT],
['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
['--filter', GetoptLong::REQUIRED_ARGUMENT],
['--verbose', '-v', GetoptLong::NO_ARGUMENT],
['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
['--debug', GetoptLong::NO_ARGUMENT],
['--release', GetoptLong::NO_ARGUMENT],
['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
| opt, arg |
case opt
when '--help'
usage
when '--jsc'
jscArg = arg
when '--no-copy'
$doNotMessWithVMPath = true
when '--output-dir'
$outputDir = Pathname.new(arg)
when '--memory-limited'
$memoryLimited = true
when '--no-jit'
$jitTests = false
when '--verbose'
$verbosity += 1
when '--run-bundle'
$bundle = Pathname.new(arg)
when '--tarball'
$tarball = true
$copyVM = true
$tarFileName = arg unless arg == ''
when '--force-vm-copy'
$copyVM = true
when '--shell-runner'
$testRunnerType = :shell
when '--make-runner'
$testRunnerType = :make
when '--remote'
$copyVM = true
$tarball = true
$remote = true
uri = URI("ssh://" + arg)
$remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
when '--remote-config-file'
$remoteConfigFile = arg
when '--child-processes'
$numChildProcesses = arg.to_i
when '--filter'
$filter = Regexp.new(arg)
when '--arch'
$architecture = arg
when '--os'
$hostOS = arg
when '--env-vars'
$envVars = arg.gsub(/\s+/, ' ').split(' ')
when '--quick'
$quickMode = true
when '--debug'
ifJSCArgIsntProvidedAreWeReleaseBuild = false
when '--release'
ifJSCArgIsntProvidedAreWeReleaseBuild = true
end
}
if $remoteConfigFile
file = File.read($remoteConfigFile)
config = JSON.parse(file)
if !$remote and config['remote']
$copyVM = true
$tarball = true
$remote = true
uri = URI("ssh://" + config['remote'])
$remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
end
if config['remoteDirectory']
$remoteDirectory = config['remoteDirectory']
end
end
unless jscArg
# If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
command = SCRIPTS_PATH.join("webkit-build-directory").to_s
command += ifJSCArgIsntProvidedAreWeReleaseBuild ? " --release" : " --debug"
command += " --executablePath"
output = `#{command}`.split("\n")
if !output.length
$stderr.puts "Error: must specify --jsc <path>"
exit 1
end
output = output[0]
jscArg = Pathname.new(output).join("jsc")
jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
if !File.file?(jscArg)
$stderr.puts "Error: must specify --jsc <path>"
exit 1
end
puts "Using the following jsc path: #{jscArg}"
end
if $doNotMessWithVMPath
$jscPath = Pathname.new(jscArg)
else
$jscPath = Pathname.new(jscArg).realpath
end
$progressMeter = ($verbosity == 0 and $stdout.tty?)
if $bundle
$jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
$outputDir = $bundle
end
# Try to determine architecture. Return nil on failure.
def machOArchitectureCode
begin
otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
otoolLines.each_with_index {
| value, index |
if value =~ /magic/ and value =~ /cputype/
return otoolLines[index + 1].split[1].to_i
end
}
rescue
$stderr.puts "Warning: unable to execute otool."
end
$stderr.puts "Warning: unable to determine architecture."
nil
end
def determineArchitectureFromMachOBinary
code = machOArchitectureCode
return nil unless code
is64BitFlag = 0x01000000
case code
when 7
"x86"
when 7 | is64BitFlag
"x86-64"
when 12
"arm"
when 12 | is64BitFlag
"arm64"
else
$stderr.puts "Warning: unable to determine architecture from code: #{code}"
nil
end
end
def determineArchitectureFromELFBinary
f = File.open($jscPath.to_s)
data = f.read(19)
if !(data[0,4] == "\x7F\x45\x4C\x46")
$stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
return nil
end
code = data[18].ord
case code
when 3
"x86"
when 62
"x86-64"
when 40
"arm"
when 183
"arm64"
else
$stderr.puts "Warning: unable to determine architecture from code: #{code}"
nil
end
end
def determineArchitectureFromPEBinary
f = File.open($jscPath.to_s)
data = f.read(1024)
if !(data[0, 2] == "MZ")
$stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
return nil
end
peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
if !(data[peHeaderAddr, 4] == "PE\0\0")
$stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
return nil
end
machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
case machine
when 0x014c
"x86"
when 0x8664
"x86-64"
else
$stderr.puts "Warning: unsupported machine type: #{machine}"
nil
end
end
def determineArchitecture
case $hostOS
when "darwin"
determineArchitectureFromMachOBinary
when "linux"
determineArchitectureFromELFBinary
when "windows"
determineArchitectureFromPEBinary
else
$stderr.puts "Warning: unable to determine architecture on this platform."
nil
end
end
def determineOS
case RbConfig::CONFIG["host_os"]
when /darwin/i
"darwin"
when /linux/i
"linux"
when /mswin|mingw|cygwin/
"windows"
else
$stderr.puts "Warning: unable to determine host operating system"
nil
end
end
$hostOS = determineOS unless $hostOS
$architecture = determineArchitecture unless $architecture
$isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $hostOS == "windows" || $hostOS == "linux" && $architecture == "arm64")
if !$testRunnerType
if $remote and $hostOS == "darwin"
$testRunnerType = :shell
else
$testRunnerType = :make
end
end
$numFailures = 0
$numPasses = 0
# We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--maxPerThreadStackUsage=1572864"]
EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
FTL_OPTIONS = ["--useFTLJIT=true"]
$runlist = []
def frameworkFromJSCPath(jscPath)
parentDirectory = jscPath.dirname
if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
parentDirectory.dirname
elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
jscPath.dirname + "JavaScriptCore.framework"
else
$stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
nil
end
end
def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
dir = Pathname.new(".")
$benchmarkDirectory.each_filename {
| pathComponent |
dir += ".."
}
dir + resourcePath
end
def pathToVM
pathToBundleResourceFromBenchmarkDirectory($jscPath)
end
def pathToHelpers
pathToBundleResourceFromBenchmarkDirectory(".helpers")
end
def prefixCommand(prefix)
"awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
end
def redirectAndPrefixCommand(prefix)
prefixCommand(prefix) + " 2>&1"
end
def pipeAndPrefixCommand(outputFilename, prefix)
"tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
end
# Output handler for tests that are expected to be silent.
def silentOutputHandler
Proc.new {
| name |
" | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
}
end
# Output handler for tests that are expected to produce meaningful output.
def noisyOutputHandler
Proc.new {
| name |
" | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
}
end
# Error handler for tests that fail exactly when they return non-zero exit status.
# This is useful when a test is expected to fail.
def simpleErrorHandler
Proc.new {
| outp, plan |
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "else"
outp.puts " " + plan.successCommand
outp.puts "fi"
}
end
# Error handler for tests that fail exactly when they return zero exit status.
def expectedFailErrorHandler
Proc.new {
| outp, plan |
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " " + plan.successCommand
outp.puts "else"
outp.puts " (echo ERROR: Unexpected exit code: 0) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "fi"
}
end
# Error handler for tests that fail exactly when they return non-zero exit status and produce
# lots of spew. This will echo that spew when the test fails.
def noisyErrorHandler
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "else"
outp.puts " " + plan.successCommand
outp.puts "fi"
}
end
# Error handler for tests that diff their output with some expectation.
def diffErrorHandler(expectedFilename)
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
outp.puts "then"
outp.puts " diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
outp.puts " if [ $? -eq 0 ]"
outp.puts " then"
outp.puts " " + plan.successCommand
outp.puts " else"
outp.puts " (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts " fi"
outp.puts "else"
outp.puts " (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "fi"
}
end
# Error handler for tests that report error by saying "failed!". This is used by Mozilla
# tests.
def mozillaErrorHandler
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "elif grep -i -q failed! #{outputFilename}"
outp.puts "then"
outp.puts " (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "else"
outp.puts " " + plan.successCommand
outp.puts "fi"
}
end
# Error handler for tests that report error by saying "failed!", and are expected to
# fail. This is used by Mozilla tests.
def mozillaFailErrorHandler
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " " + plan.successCommand
outp.puts "elif grep -i -q failed! #{outputFilename}"
outp.puts "then"
outp.puts " " + plan.successCommand
outp.puts "else"
outp.puts " (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "fi"
}
end
# Error handler for tests that report error by saying "failed!", and are expected to have
# an exit code of 3.
def mozillaExit3ErrorHandler
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " if [ `cat #{plan.failFile}` -eq 3 ]"
outp.puts " then"
outp.puts " if grep -i -q failed! #{outputFilename}"
outp.puts " then"
outp.puts " (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts " else"
outp.puts " " + plan.successCommand
outp.puts " fi"
outp.puts " else"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts " fi"
outp.puts "else"
outp.puts " (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "fi"
}
end
# Error handler for tests that report success by saying "Passed" or error by saying "FAILED".
# This is used by Chakra tests.
def chakraPassFailErrorHandler
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "elif grep -i -q FAILED #{outputFilename}"
outp.puts "then"
outp.puts " (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "else"
outp.puts " " + plan.successCommand
outp.puts "fi"
}
end
$runCommandOptions = {}
class Plan
attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler
attr_accessor :index
def initialize(directory, arguments, family, name, outputHandler, errorHandler)
@directory = directory
@arguments = arguments
@family = family
@name = name
@outputHandler = outputHandler
@errorHandler = errorHandler
@isSlow = !!$runCommandOptions[:isSlow]
end
def shellCommand
# It's important to remember that the test is actually run in a subshell, so if we change directory
# in the subshell when we return we will be in our original directory. This is nice because we don't
# have to bend over backwards to do things relative to the root.
script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
$envVars.each { |var| script += "export " << var << "; " }
script += "\"$@\" " + escapeAll(@arguments) + "))"
return script
end
def reproScriptCommand
# We have to find our way back to the .runner directory since that's where all of the relative
# paths assume they start out from.
script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
script += "cd $CURRENT_DIR\n"
Pathname.new(@name).dirname.each_filename {
| pathComponent |
script += "cd ..\n"
}
script += "cd .runner\n"
script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
script += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])}\n"
$envVars.each { |var| script += "export " << var << "\n" }
script += "#{shellCommand} || exit 1"
"echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
end
def failCommand
"echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
end
def successCommand
if $progressMeter or $verbosity >= 2
"rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
else
"rm -f #{failFile}"
end
end
def failFile
"test_fail_#{@index}"
end
def writeRunScript(filename)
File.open(filename, "w") {
| outp |
outp.puts "echo Running #{Shellwords.shellescape(@name)}"
cmd = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
cmd += @outputHandler.call(@name)
if $verbosity >= 3
outp.puts "echo #{Shellwords.shellescape(cmd)}"
end
outp.puts cmd
@errorHandler.call(outp, self)
}
end
end
$uniqueFilenameCounter = 0
def uniqueFilename(extension)
payloadDir = $outputDir + "_payload"
Dir.mkdir payloadDir unless payloadDir.directory?
result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
$uniqueFilenameCounter += 1
result
end
def baseOutputName(kind)
"#{$collectionName}/#{$benchmark}.#{kind}"
end
def addRunCommand(kind, command, outputHandler, errorHandler)
$didAddRunCommand = true
name = baseOutputName(kind)
if $filter and name !~ $filter
return
end
plan = Plan.new(
$benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
errorHandler)
if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
$runlist.unshift plan
else
$runlist << plan
end
end
# Returns true if there were run commands found in the file ($benchmarkDirectory +
# $benchmark), in which case those run commands have already been executed. Otherwise
# returns false, in which case you're supposed to add your own run commands.
def parseRunCommands
oldDidAddRunCommand = $didAddRunCommand
$didAddRunCommand = false
Dir.chdir($outputDir) {
File.open($benchmarkDirectory + $benchmark) {
| inp |
inp.each_line {
| line |
begin
doesMatch = line =~ /^\/\/@/
rescue Exception => e
# Apparently this happens in the case of some UTF8 stuff in some files, where
# Ruby tries to be strict and throw exceptions.
next
end
next unless doesMatch
eval $~.post_match
}
}
}
result = $didAddRunCommand
$didAddRunCommand = result or oldDidAddRunCommand
result
end
def slow!
$runCommandOptions[:isSlow] = true
end
def run(kind, *options)
addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
end
def runNoFTL
run("no-ftl")
end
def runWithRAMSize(size)
run("ram-size-#{size}", "--forceRAMSize=#{size}")
end
def runOneLargeHeap
if $memoryLimited
$didAddRunCommand = true
puts "Skipping #{$collectionName}/#{$benchmark}"
else
run("default")
end
end
def runNoJIT
run("no-jit", "--useJIT=false")
end
def runNoLLInt
if $jitTests
run("no-llint", "--useLLInt=false")
end
end
def runNoCJITValidate
run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
end
def runNoCJITValidatePhases
run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *NO_CJIT_OPTIONS)
end
def runDefault
run("default", *FTL_OPTIONS)
end
def runFTLNoCJIT
run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end
def runFTLNoCJITValidate
run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end
def runFTLNoCJITNoPutStackValidate
run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end
def runFTLNoCJITNoInlineValidate
run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end
def runFTLNoCJITOSRValidation
run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end
def runDFGEager
run("dfg-eager", *EAGER_OPTIONS)
end
def runDFGEagerNoCJITValidate
run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
end
def runFTLEager
run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS))
end
def runFTLEagerNoCJITValidate
run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
end
def runFTLEagerNoCJITOSRValidation
run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
end
def runNoCJITNoASO
run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
end
def runNoCJITNoAccessInlining
run("no-cjit-no-access-inlining", "--useAccessInlining=false", *NO_CJIT_OPTIONS)
end
def runFTLNoCJITNoAccessInlining
run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end
def runFTLNoCJITSmallPool
run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end
def runMiscNoCJITTest(*options)
run("misc-no-cjit", *(NO_CJIT_OPTIONS + options))
end
def runMiscFTLNoCJITTest(*options)
run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + options))
end
def runDFGMaximalFlushPhase
run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *NO_CJIT_OPTIONS)
end
def runShadowChicken
run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true")
end
def defaultRun
if $quickMode
defaultQuickRun
else
runDefault
if $jitTests
runNoLLInt
runNoCJITValidatePhases
runDFGEager
runDFGEagerNoCJITValidate
runDFGMaximalFlushPhase
return if !$isFTLPlatform
runNoFTL
runFTLNoCJITValidate
runFTLNoCJITNoPutStackValidate
runFTLNoCJITNoInlineValidate
runFTLEager
runFTLEagerNoCJITValidate
runFTLNoCJITSmallPool
end
end
end
def defaultNoNoLLIntRun
if $quickMode
defaultQuickRun
else
runDefault
if $jitTests
runNoCJITValidatePhases
runDFGEager
runDFGEagerNoCJITValidate
runDFGMaximalFlushPhase
return if !$isFTLPlatform
runNoFTL
runFTLNoCJITValidate
runFTLNoCJITNoPutStackValidate
runFTLNoCJITNoInlineValidate
runFTLEager
runFTLEagerNoCJITValidate
runFTLNoCJITSmallPool
end
end
end
def defaultQuickRun
runDefault
if $jitTests
runNoCJITValidate
return if $isFTLPlatform
runNoFTL
runFTLNoCJITValidate
end
end
def defaultSpotCheckNoMaximalFlush
defaultQuickRun
runNoCJITNoAccessInlining
return if !$isFTLPlatform
runFTLNoCJITOSRValidation
runFTLNoCJITNoAccessInlining
end
def defaultSpotCheck
defaultSpotCheckNoMaximalFlush
runDFGMaximalFlushPhase
end
# This is expected to not do eager runs because eager runs can have a lot of recompilations
# for reasons that don't arise in the real world. It's used for tests that assert convergence
# by counting recompilations.
def defaultNoEagerRun
runDefault
if $jitTests
runNoLLInt
runNoCJITValidatePhases
return if !$isFTLPlatform
runNoFTL
runFTLNoCJITValidate
runFTLNoCJITNoInlineValidate
end
end
def defaultNoSamplingProfilerRun
runDefault
if $jitTests
runNoLLInt
runNoCJITValidatePhases
runDFGEager
runDFGEagerNoCJITValidate
runDFGMaximalFlushPhase
return if !$isFTLPlatform
runNoFTL
runFTLNoCJITNoPutStackValidate
runFTLNoCJITNoInlineValidate
runFTLEager
runFTLEagerNoCJITValidate
runFTLNoCJITSmallPool
end
end
def runProfiler
if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
skip
return
end
profilerOutput = uniqueFilename(".json")
if $canRunDisplayProfilerOutput
addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
else
puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
end
end
def runExceptionFuzz
subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
end
def runExecutableAllocationFuzz(name, *options)
subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
end
def runTypeProfiler
if !$jitTests
return
end
return if !$isFTLPlatform
run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
end
def runControlFlowProfiler
if !$jitTests
return
end
return if !$isFTLPlatform
run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end
def runTest262(mode, exception, includeFiles, flags)
failsWithException = exception != "NoException"
isStrict = false
isModule = false
isAsync = false
flags.each {
| flag |
case flag
when :strict
isStrict = true
when :module
isModule = true
when :async
isAsync = true
else
raise "Invalid flag for runTest262, #{flag}"
end
}
prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
args = [pathToVM.to_s] + BASE_OPTIONS
args << "--exception=" + exception if failsWithException
args << "--test262-async" if isAsync
args += includeFiles
case mode
when :normal
errorHandler = simpleErrorHandler
outputHandler = silentOutputHandler
when :fail
errorHandler = expectedFailErrorHandler
outputHandler = noisyOutputHandler
else
raise "Invalid mode: #{mode}"
end
if isStrict
kind = "default-strict"
args << "--strict-file=#{$benchmark}"
else
kind = "default"
if isModule
args << "--module-file=#{$benchmark}"
else
args << $benchmark.to_s
end
end
addRunCommand(kind, args, outputHandler, errorHandler)
end
def prepareTest262Fixture
# This function is used to add the files used by Test262 modules tests.
prepareExtraRelativeFiles([""], $collection)
end
def runES6(mode)
args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
case mode
when :normal
errorHandler = simpleErrorHandler
when :fail
errorHandler = expectedFailErrorHandler
else
raise "Invalid mode: #{mode}"
end
addRunCommand("default", args, noisyOutputHandler, errorHandler)
end
def runModules
run("default-modules", "-m")
if !$jitTests
return
end
run("no-llint-modules", "-m", "--useLLInt=false")
run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
return if !$isFTLPlatform
run("default-ftl-modules", "-m", *FTL_OPTIONS)
run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end
def runChakra(mode, exception, baselineFile, extraFiles)
raise unless $benchmark.to_s =~ /\.js$/
failsWithException = exception != "NoException"
testName = $~.pre_match
prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
args = [pathToVM.to_s] + BASE_OPTIONS
args += FTL_OPTIONS if $isFTLPlatform
args += EAGER_OPTIONS
args << "--exception=" + exception if failsWithException
args << "--dumpException" if failsWithException
args += ["jsc-lib.js"]
case mode
when :baseline
prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
outputHandler = noisyOutputHandler
when :pass
errorHandler = chakraPassFailErrorHandler
outputHandler = noisyOutputHandler
when :skip
return
else
raise "Invalid mode: #{mode}"
end
kind = "default"
args << $benchmark.to_s
addRunCommand(kind, args, outputHandler, errorHandler)
end
def runLayoutTest(kind, *options)
raise unless $benchmark.to_s =~ /\.js$/
testName = $~.pre_match
if kind
kind = "layout-" + kind
else
kind = "layout"
end
prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
args = [pathToVM.to_s] + BASE_OPTIONS + options +
[(Pathname.new("resources") + "standalone-pre.js").to_s,
$benchmark.to_s,
(Pathname.new("resources") + "standalone-post.js").to_s]
addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
end
def runLayoutTestNoFTL
runLayoutTest("no-ftl")
end
def runLayoutTestNoLLInt
runLayoutTest("no-llint", "--useLLInt=false")
end
def runLayoutTestNoCJIT
runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
end
def runLayoutTestDFGEagerNoCJIT
runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
end
def runLayoutTestDefault
runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
end
def runLayoutTestFTLNoCJIT
runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end
def runLayoutTestFTLEagerNoCJIT
runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
end
def noFTLRunLayoutTest
if !$jitTests
return
end
runLayoutTestNoLLInt
runLayoutTestNoCJIT
runLayoutTestDFGEagerNoCJIT
end
def defaultQuickRunLayoutTest
runLayoutTestDefault
if $jitTests
if $isFTLPlatform
runLayoutTestNoFTL
runLayoutTestFTLNoCJIT
runLayoutTestFTLEagerNoCJIT
else
noFTLRunLayoutTest
end
end
end
def defaultRunLayoutTest
if $quickMode
defaultQuickRunLayoutTest
else
runLayoutTestDefault
if $jitTests
noFTLRunLayoutTest
return if !$isFTLPlatform
runLayoutTestNoFTL
runLayoutTestFTLNoCJIT
runLayoutTestFTLEagerNoCJIT
end
end
end
def noEagerNoNoLLIntTestsRunLayoutTest
runLayoutTestDefault
if $jitTests
runLayoutTestNoCJIT
return if !$isFTLPlatform
runLayoutTestNoFTL
runLayoutTestFTLNoCJIT
end
end
def noNoLLIntRunLayoutTest
runLayoutTestDefault
if $jitTests
runLayoutTestNoCJIT
runLayoutTestDFGEagerNoCJIT
return if !$isFTLPlatform
runLayoutTestNoFTL
runLayoutTestFTLNoCJIT
runLayoutTestFTLEagerNoCJIT
end
end
def prepareExtraRelativeFiles(extraFiles, destination)
Dir.chdir($outputDir) {
extraFiles.each {
| file |
dest = destination + file
FileUtils.mkdir_p(dest.dirname)
FileUtils.cp $extraFilesBaseDir + file, dest
}
}
end
def baseDirForCollection(collectionName)
Pathname(".tests") + collectionName
end
def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
raise unless absoluteBase.absolute?
Dir.chdir($outputDir) {
collectionBaseDir = baseDirForCollection($collectionName)
extraFiles.each {
| file |
destination = collectionBaseDir + file
FileUtils.mkdir_p destination.dirname unless destination.directory?
FileUtils.cp absoluteBase + file, destination
}
}
end
def runMozillaTest(kind, mode, extraFiles, *options)
if kind
kind = "mozilla-" + kind
else
kind = "mozilla"
end
prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
case mode
when :normal
errorHandler = mozillaErrorHandler
when :negative
errorHandler = mozillaExit3ErrorHandler
when :fail
errorHandler = mozillaFailErrorHandler
when :skip
return
else
raise "Invalid mode: #{mode}"
end
addRunCommand(kind, args, noisyOutputHandler, errorHandler)
end
def runMozillaTestDefault(mode, *extraFiles)
runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
end
def runMozillaTestNoFTL(mode, *extraFiles)
runMozillaTest("no-ftl", mode, extraFiles)
end
def runMozillaTestLLInt(mode, *extraFiles)
runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
end
def runMozillaTestBaselineJIT(mode, *extraFiles)
runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
end
def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
end
def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
end
def defaultQuickRunMozillaTest(mode, *extraFiles)
if $jitTests
runMozillaTestDefault(mode, *extraFiles)
runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
else
runMozillaTestNoFTL(mode, *extraFiles)
if $jitTests
runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
end
end
end
def defaultRunMozillaTest(mode, *extraFiles)
if $quickMode
defaultQuickRunMozillaTest(mode, *extraFiles)
else
runMozillaTestNoFTL(mode, *extraFiles)
if $jitTests
runMozillaTestLLInt(mode, *extraFiles)
runMozillaTestBaselineJIT(mode, *extraFiles)
runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
runMozillaTestDefault(mode, *extraFiles)
runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
end
end
end
def runNoisyTest(kind, *options)
addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
end
def runNoisyTestDefault
runNoisyTest("default", *FTL_OPTIONS)
end
def runNoisyTestNoFTL
runNoisyTest("no-ftl")
end
def runNoisyTestNoCJIT
runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end
def runNoisyTestEagerNoCJIT
runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
end
def defaultRunNoisyTest
runNoisyTestDefault
if $jitTests and $isFTLPlatform
runNoisyTestNoFTL
runNoisyTestNoCJIT
runNoisyTestEagerNoCJIT
end
end
def skip
$didAddRunCommand = true
puts "Skipping #{$collectionName}/#{$benchmark}"
end
def largeHeap
if $memoryLimited
$didAddRunCommand = true
puts "Skipping #{$collectionName}/#{$benchmark}"
end
end
def allJSFiles(path)
if path.file?
[path]
else
result = []
Dir.foreach(path) {
| filename |
next unless filename =~ /\.js$/
next unless (path + filename).file?
result << path + filename
}
result
end
end
def uniqueifyName(names, name)
result = name.to_s
toAdd = 1
while names[result]
result = "#{name}-#{toAdd}"
toAdd += 1
end
names[result] = true
result
end
def simplifyCollectionName(collectionPath)
outerDir = collectionPath.dirname
name = collectionPath.basename
lastName = name
if collectionPath.directory?
while lastName.to_s =~ /test/
lastName = outerDir.basename
name = lastName + name
outerDir = outerDir.dirname
end
end
uniqueifyName($collectionNames, name)
end
def prepareCollection(name)
FileUtils.mkdir_p $outputDir + name
absoluteCollection = $collection.realpath
Dir.chdir($outputDir) {
bundleDir = baseDirForCollection(name)
# Create the proper directory structures.
FileUtils.mkdir_p bundleDir
if bundleDir.basename == $collection.basename
FileUtils.cp_r absoluteCollection, bundleDir.dirname
$collection = bundleDir
else
FileUtils.cp_r absoluteCollection, bundleDir
$collection = bundleDir + $collection.basename
end
$extraFilesBaseDir = absoluteCollection
}
end
$collectionNames = {}
def handleCollectionFile(collection)
collectionName = simplifyCollectionName(collection)
paths = {}
subCollections = []
YAML::load(IO::read(collection)).each {
| entry |
if entry["collection"]
subCollections << entry["collection"]
next
end
if Pathname.new(entry["path"]).absolute?
raise "Absolute path: " + entry["path"] + " in #{collection}"
end
if paths[entry["path"]]
raise "Duplicate path: " + entry["path"] + " in #{collection}"
end
subCollection = collection.dirname + entry["path"]
if subCollection.file?
subCollectionName = Pathname.new(entry["path"]).dirname
else
subCollectionName = entry["path"]
end
$collection = subCollection
$collectionName = Pathname.new(collectionName)
Pathname.new(subCollectionName).each_filename {
| filename |
next if filename =~ /^\./
$collectionName += filename
}
$collectionName = $collectionName.to_s
prepareCollection($collectionName)
Dir.chdir($outputDir) {
pathsToSearch = [$collection]
if entry["tests"]
if entry["tests"].is_a? Array
pathsToSearch = entry["tests"].map {
| testName |
pathsToSearch[0] + testName
}
else
pathsToSearch[0] += entry["tests"]
end
end
pathsToSearch.each {
| pathToSearch |
allJSFiles(pathToSearch).each {
| path |
$benchmark = path.basename
$benchmarkDirectory = path.dirname
$runCommandOptions = {}
eval entry["cmd"]
}
}
}
}
subCollections.each {
| subCollection |
handleCollection(collection.dirname + subCollection)
}
end
def handleCollectionDirectory(collection)
collectionName = simplifyCollectionName(collection)
$collection = collection
$collectionName = collectionName
prepareCollection(collectionName)
Dir.chdir($outputDir) {
$benchmarkDirectory = $collection
allJSFiles($collection).each {
| path |
$benchmark = path.basename
$runCommandOptions = {}
defaultRun unless parseRunCommands
}
}
end
def handleCollection(collection)
collection = Pathname.new(collection)
if collection.file?
handleCollectionFile(collection)
else
handleCollectionDirectory(collection)
end
end
def appendFailure(plan)
File.open($outputDir + "failed", "a") {
| outp |
outp.puts plan.name
}
$numFailures += 1
end
def appendPass(plan)
File.open($outputDir + "passed", "a") {
| outp |
outp.puts plan.name
}
$numPasses += 1
end
def appendResult(plan, didPass)
File.open($outputDir + "results", "a") {
| outp |
outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
}
end
def prepareBundle
raise if $bundle
if $doNotMessWithVMPath
if !$remote and !$tarball
$testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
$jscPath = Pathname.new($jscPath).realpath
else
$testingFrameworkPath = frameworkFromJSCPath($jscPath)
end
else
originalJSCPath = $jscPath
vmDir = $outputDir + ".vm"
FileUtils.mkdir_p vmDir
frameworkPath = frameworkFromJSCPath($jscPath)
destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
$jscPath = destinationFrameworkPath + "Resources" + "jsc"
$testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
if frameworkPath
source = frameworkPath
destination = Pathname.new(".vm")
else
source = originalJSCPath
destination = $jscPath
Dir.chdir($outputDir) {
FileUtils.mkdir_p $jscPath.dirname
}
end
Dir.chdir($outputDir) {
if $copyVM
FileUtils.cp_r source, destination
else
begin
FileUtils.ln_s source, destination
rescue Exception
$stderr.puts "Warning: unable to create soft link, trying to copy."
FileUtils.cp_r source, destination
end
end
if $remote and $hostOS == "linux"
begin
dependencies = `ldd #{source}`
dependencies.split(/\n/).each {
| dependency |
FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
}
rescue
$stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
end
end
}
end
Dir.chdir($outputDir) {
FileUtils.cp_r HELPERS_PATH, ".helpers"
}
ARGV.each {
| collection |
handleCollection(collection)
}
puts
end
def cleanOldResults
raise unless $bundle
eachResultFile($outputDir) {
| path |
FileUtils.rm_f path
}
end
def cleanEmptyResultFiles
eachResultFile($outputDir) {
| path |
next unless path.basename.to_s =~ /\.out$/
next unless FileTest.size(path) == 0
FileUtils.rm_f path
}
end
def eachResultFile(startingDir, &block)
dirsToClean = [startingDir]
until dirsToClean.empty?
nextDir = dirsToClean.pop
Dir.foreach(nextDir) {
| entry |
next if entry =~ /^\./
path = nextDir + entry
if path.directory?
dirsToClean.push(path)
else
block.call(path)
end
}
end
end
def prepareTestRunner
raise if $bundle
$runlist.each_with_index {
| plan, index |
plan.index = index
}
Dir.mkdir($runnerDir) unless $runnerDir.directory?
toDelete = []
Dir.foreach($runnerDir) {
| filename |
if filename =~ /^test_/
toDelete << filename
end
}
toDelete.each {
| filename |
File.unlink($runnerDir + filename)
}
$runlist.each {
| plan |
plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
}
case $testRunnerType
when :make
prepareMakeTestRunner
when :shell
prepareShellTestRunner
else
raise "Unknown test runner type: #{$testRunnerType.to_s}"
end
end
def prepareShellTestRunner
FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
end
def prepareMakeTestRunner
# The goals of our parallel test runner are scalability and simplicity. The
# simplicity part is particularly important. We don't want to have to have
# a full-time contributor just philosophising about parallel testing.
#
# As such, we just pass off all of the hard work to 'make'. This creates a
# dummy directory ("$outputDir/.runner") in which we create a dummy
# Makefile. The Makefile has an 'all' rule that depends on all of the tests.
# That is, for each test we know we will run, there is a rule in the
# Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
# Makefile results in 'make' doing all of the hard work:
#
# - Load balancing just works. Most systems have a great load balancer in
# 'make'. If your system doesn't then just install a real 'make'.
#
# - Interruptions just work. For example Ctrl-C handling in 'make' is
# exactly right. You don't have to worry about zombie processes.
#
# We then do some tricks to make failure detection work and to make this
# totally sound. If a test fails, we don't want the whole 'make' job to
# stop. We also don't have any facility for makefile-escaping of path names.
# We do have such a thing for shell-escaping, though. We fix both problems
# by having the actual work for each of the test rules be done in a shell
# script on the side. There is one such script per test. The script responds
# to failure by printing something on the console and then touching a
# failure file for that test, but then still returns 0. This makes 'make'
# continue past that failure and complete all the tests anyway.
#
# In the end, this script collects all of the failures by searching for
# files in the .runner directory whose name matches /^test_fail_/, where
# the thing after the 'fail_' is the test index. Those are the files that
# would be created by the test scripts if they detect failure. We're
# basically using the filesystem as a concurrent database of test failures.
# Even if two tests fail at the same time, since they're touching different
# files we won't miss any failures.
runIndices = []
$runlist.each {
| plan |
runIndices << plan.index
}
File.open($runnerDir + "Makefile", "w") {
| outp |
outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
runIndices.each {
| index |
plan = $runlist[index]
outp.puts "test_done_#{index}:"
outp.puts "\tsh test_script_#{plan.index}"
}
}
end
def cleanRunnerDirectory
raise unless $bundle
Dir.foreach($runnerDir) {
| filename |
next unless filename =~ /^test_fail/
FileUtils.rm_f $runnerDir + filename
}
end
def sshRead(cmd)
raise unless $remote
result = ""
IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
| inp |
inp.each_line {
| line |
result += line
}
}
raise "#{$?}" unless $?.success?
result
end
def runCommandOnTester(cmd)
if $remote
result = sshRead(cmd)
else
result = `#{cmd}`
end
end
def numberOfProcessors
if $hostOS == "windows"
numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
else
begin
numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
rescue
numProcessors = 0
end
if numProcessors == 0
begin
numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
rescue
numProcessors == 0
end
end
end
if numProcessors == 0
numProcessors = 1
end
return numProcessors
end
def runAndMonitorTestRunnerCommand(*cmd)
numberOfTests = 0
Dir.chdir($runnerDir) {
# -1 for the runscript, and -2 for '..' and '.'
numberOfTests = Dir.entries(".").count - 3
}
unless $progressMeter
mysys(cmd.join(' '))
else
running = {}
didRun = {}
didFail = {}
blankLine = true
prevStringLength = 0
IO.popen(cmd.join(' '), mode="r") {
| inp |
inp.each_line {
| line |
line = line.scrub.chomp
if line =~ /^Running /
running[$~.post_match] = true
elsif line =~ /^PASS: /
didRun[$~.post_match] = true
elsif line =~ /^FAIL: /
didRun[$~.post_match] = true
didFail[$~.post_match] = true
else
unless blankLine
print("\r" + " " * prevStringLength + "\r")
end
puts line
blankLine = true
end
def lpad(str, chars)
str = str.to_s
if str.length > chars
str
else
"%#{chars}s"%(str)
end
end
string = ""
string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
unless didFail.empty?
string += " (failed #{didFail.size})"
end
string += " "
(running.size - didRun.size).times {
string += "."
}
if string.length < prevStringLength
print string
print(" " * (prevStringLength - string.length))
end
print string
prevStringLength = string.length
blankLine = false
$stdout.flush
}
}
puts
raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
end
end
def runTestRunner
case $testRunnerType
when :shell
testRunnerCommand = "sh runscript"
when :make
testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
else
raise "Unknown test runner type: #{$testRunnerType.to_s}"
end
if $remote
if !$remoteDirectory
$remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
end
mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
remoteScript = "\""
remoteScript += "cd #{$remoteDirectory} && "
remoteScript += "rm -rf #{$outputDir.basename} && "
remoteScript += "tar xzf #{$tarFileName} && "
remoteScript += "cd #{$outputDir.basename}/.runner && "
remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
$envVars.each { |var| remoteScript += "export " << var << "\n" }
remoteScript += "#{testRunnerCommand}\""
runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
else
Dir.chdir($runnerDir) {
runAndMonitorTestRunnerCommand(testRunnerCommand)
}
end
end
def detectFailures
raise if $bundle
failures = []
if $remote
output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
output.split(/\n/).each {
| line |
next unless line =~ /test_fail_/
failures << $~.post_match.to_i
}
else
Dir.foreach($runnerDir) {
| filename |
next unless filename =~ /test_fail_/
failures << $~.post_match.to_i
}
end
failureSet = {}
failures.each {
| failure |
appendFailure($runlist[failure])
failureSet[failure] = true
}
familyMap = {}
$runlist.each_with_index {
| plan, index |
unless familyMap[plan.family]
familyMap[plan.family] = []
end
if failureSet[index]
appendResult(plan, false)
familyMap[plan.family] << {:result => "FAIL", :plan => plan};
next
else
appendResult(plan, true)
familyMap[plan.family] << {:result => "PASS", :plan => plan};
end
appendPass(plan)
}
File.open($outputDir + "resultsByFamily", "w") {
| outp |
first = true
familyMap.keys.sort.each {
| familyName |
if first
first = false
else
outp.puts
end
outp.print "#{familyName}:"
numPassed = 0
familyMap[familyName].each {
| entry |
if entry[:result] == "PASS"
numPassed += 1
end
}
if numPassed == familyMap[familyName].size
outp.puts " PASSED"
elsif numPassed == 0
outp.puts " FAILED"
else
outp.puts
familyMap[familyName].each {
| entry |
outp.puts " #{entry[:plan].name}: #{entry[:result]}"
}
end
}
}
end
def compressBundle
cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
$stderr.puts ">> #{cmd}" if $verbosity >= 2
raise unless system(cmd)
end
def clean(file)
FileUtils.rm_rf file unless $bundle
end
clean($outputDir + "failed")
clean($outputDir + "passed")
clean($outputDir + "results")
clean($outputDir + "resultsByFamily")
clean($outputDir + ".vm")
clean($outputDir + ".helpers")
clean($outputDir + ".runner")
clean($outputDir + ".tests")
clean($outputDir + "_payload")
Dir.mkdir($outputDir) unless $outputDir.directory?
$outputDir = $outputDir.realpath
$runnerDir = $outputDir + ".runner"
if !$numChildProcesses
if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
$numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
else
$numChildProcesses = numberOfProcessors
end
end
if ENV["JSCTEST_timeout"]
# In the worst case, the processors just interfere with each other.
# Increase the timeout proportionally to the number of processors.
ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
end
def runBundle
raise unless $bundle
cleanRunnerDirectory
cleanOldResults
runTestRunner
cleanEmptyResultFiles
end
def runNormal
raise if $bundle or $tarball
prepareBundle
prepareTestRunner
runTestRunner
cleanEmptyResultFiles
detectFailures
end
def runTarball
raise unless $tarball
prepareBundle
prepareTestRunner
compressBundle
end
def runRemote
raise unless $remote
prepareBundle
prepareTestRunner
compressBundle
runTestRunner
detectFailures
end
puts
if $bundle
runBundle
elsif $remote
runRemote
elsif $tarball
runTarball
else
runNormal
end