blob: aafa33a3dfa2e53d9fe248cea9a1dc1018312b27 [file] [log] [blame]
// RUN: %{swiftc} %s -o %T/Performance
// RUN: %T/Performance > %t || true
// RUN: %{xctest_checker} %t %s
#if os(macOS)
import SwiftXCTest
#else
import XCTest
#endif
// CHECK: Test Suite 'All tests' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: Test Suite '.*\.xctest' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: Test Suite 'PerformanceTestCase' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
class PerformanceTestCase: XCTestCase {
// CHECK: Test Case 'PerformanceTestCase.test_measureBlockIteratesTenTimes' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+4]]: Test Case 'PerformanceTestCase.test_measureBlockIteratesTenTimes' measured \[Time, seconds\] .*
// CHECK: Test Case 'PerformanceTestCase.test_measureBlockIteratesTenTimes' passed \(\d+\.\d+ seconds\)
func test_measureBlockIteratesTenTimes() {
var iterationCount = 0
measure(block: {
iterationCount += 1
})
XCTAssertEqual(iterationCount, 10)
}
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithAutomaticStartAndStop' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+4]]: Test Case 'PerformanceTestCase.test_measuresMetricsWithAutomaticStartAndStop' measured \[Time, seconds\] .*
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithAutomaticStartAndStop' passed \(\d+\.\d+ seconds\)
func test_measuresMetricsWithAutomaticStartAndStop() {
var iterationCount = 0
measureMetrics([.wallClockTime], automaticallyStartMeasuring: true, for: {
iterationCount += 1
})
XCTAssertEqual(iterationCount, 10)
}
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithManualStartAndStop' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+3]]: Test Case 'PerformanceTestCase.test_measuresMetricsWithManualStartAndStop' measured \[Time, seconds\] .*
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithManualStartAndStop' passed \(\d+\.\d+ seconds\)
func test_measuresMetricsWithManualStartAndStop() {
measureMetrics([.wallClockTime], automaticallyStartMeasuring: false) {
self.startMeasuring()
self.stopMeasuring()
}
}
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithoutExplicitStop' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+3]]: Test Case 'PerformanceTestCase.test_measuresMetricsWithoutExplicitStop' measured \[Time, seconds\] .*
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithoutExplicitStop' passed \(\d+\.\d+ seconds\)
func test_measuresMetricsWithoutExplicitStop() {
measureMetrics([.wallClockTime], automaticallyStartMeasuring: false) {
self.startMeasuring()
}
}
// CHECK: Test Case 'PerformanceTestCase.test_hasWallClockAsDefaultPerformanceMetric' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: Test Case 'PerformanceTestCase.test_hasWallClockAsDefaultPerformanceMetric' passed \(\d+\.\d+ seconds\)
func test_hasWallClockAsDefaultPerformanceMetric() {
XCTAssertEqual(PerformanceTestCase.defaultPerformanceMetrics, [.wallClockTime])
}
// CHECK: Test Case 'PerformanceTestCase.test_printsValuesAfterMeasuring' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+3]]: Test Case 'PerformanceTestCase.test_printsValuesAfterMeasuring' measured \[Time, seconds\] average: \d+.\d{3}, relative standard deviation: \d+.\d{3}%, values: \[\d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}\], performanceMetricID:org.swift.XCTPerformanceMetric_WallClockTime, maxPercentRelativeStandardDeviation: \d+.\d{3}%, maxStandardDeviation: \d.\d{3}
// CHECK: Test Case 'PerformanceTestCase.test_printsValuesAfterMeasuring' passed \(\d+\.\d+ seconds\)
func test_printsValuesAfterMeasuring() {
measure {
Thread.sleep(forTimeInterval: 1)
}
}
// CHECK: Test Case 'PerformanceTestCase.test_abortsMeasurementsAfterTestFailure' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
func test_abortsMeasurementsAfterTestFailure() {
var iterationCount = 0
measure {
iterationCount += 1
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+1]]: error: PerformanceTestCase.test_abortsMeasurementsAfterTestFailure : XCTAssertLessThan failed: \("3"\) is not less than \("3"\) -
XCTAssertLessThan(iterationCount, 3)
}
XCTAssertEqual(iterationCount, 3)
}
// CHECK: Test Case 'PerformanceTestCase.test_abortsMeasurementsAfterTestFailure' failed \(\d+\.\d+ seconds\)
// CHECK: Test Case 'PerformanceTestCase.test_measuresWallClockTimeInBlock' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
func test_measuresWallClockTimeInBlock() {
var hasWaited = false
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+2]]: Test Case 'PerformanceTestCase.test_measuresWallClockTimeInBlock' measured \[Time, seconds\] average: \d+.\d{3}, relative standard deviation: \d+.\d{3}%, values: \[1.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}\], performanceMetricID:org.swift.XCTPerformanceMetric_WallClockTime, maxPercentRelativeStandardDeviation: \d+.\d{3}%, maxStandardDeviation: \d.\d{3}
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+1]]: error: PerformanceTestCase.test_measuresWallClockTimeInBlock : failed: The relative standard deviation of the measurements is \d+.\d{3}% which is higher than the max allowed of \d+.\d{3}%.
measure {
if !hasWaited {
Thread.sleep(forTimeInterval: 1)
hasWaited = true
}
}
}
// CHECK: Test Case 'PerformanceTestCase.test_measuresWallClockTimeInBlock' failed \(\d+\.\d+ seconds\)
static var allTests = {
return [
("test_measureBlockIteratesTenTimes", test_measureBlockIteratesTenTimes),
("test_measuresMetricsWithAutomaticStartAndStop", test_measuresMetricsWithAutomaticStartAndStop),
("test_measuresMetricsWithManualStartAndStop", test_measuresMetricsWithManualStartAndStop),
("test_measuresMetricsWithoutExplicitStop", test_measuresMetricsWithoutExplicitStop),
("test_hasWallClockAsDefaultPerformanceMetric", test_hasWallClockAsDefaultPerformanceMetric),
("test_printsValuesAfterMeasuring", test_printsValuesAfterMeasuring),
("test_abortsMeasurementsAfterTestFailure", test_abortsMeasurementsAfterTestFailure),
("test_measuresWallClockTimeInBlock", test_measuresWallClockTimeInBlock),
]
}()
}
// CHECK: Test Suite 'PerformanceTestCase' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: \t Executed \d+ tests, with 2 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
XCTMain([testCase(PerformanceTestCase.allTests)])
// CHECK: Test Suite '.*\.xctest' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: \t Executed \d+ tests, with \d failures? \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
// CHECK: Test Suite 'All tests' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: \t Executed \d+ tests, with \d failures? \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds