| // RUN: %{swiftc} %s -o %T/Performance |
| // RUN: %T/Performance > %t || true |
| // RUN: %{xctest_checker} %t %s |
| |
| #if os(macOS) |
| import SwiftXCTest |
| #else |
| import XCTest |
| #endif |
| |
| // CHECK: Test Suite 'All tests' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| // CHECK: Test Suite '.*\.xctest' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| |
| // CHECK: Test Suite 'PerformanceTestCase' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| class PerformanceTestCase: XCTestCase { |
| |
| // CHECK: Test Case 'PerformanceTestCase.test_measureBlockIteratesTenTimes' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| // CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+4]]: Test Case 'PerformanceTestCase.test_measureBlockIteratesTenTimes' measured \[Time, seconds\] .* |
| // CHECK: Test Case 'PerformanceTestCase.test_measureBlockIteratesTenTimes' passed \(\d+\.\d+ seconds\) |
| func test_measureBlockIteratesTenTimes() { |
| var iterationCount = 0 |
| measure(block: { |
| iterationCount += 1 |
| }) |
| XCTAssertEqual(iterationCount, 10) |
| } |
| |
| // CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithAutomaticStartAndStop' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| // CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+4]]: Test Case 'PerformanceTestCase.test_measuresMetricsWithAutomaticStartAndStop' measured \[Time, seconds\] .* |
| // CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithAutomaticStartAndStop' passed \(\d+\.\d+ seconds\) |
| func test_measuresMetricsWithAutomaticStartAndStop() { |
| var iterationCount = 0 |
| measureMetrics([.wallClockTime], automaticallyStartMeasuring: true, for: { |
| iterationCount += 1 |
| }) |
| XCTAssertEqual(iterationCount, 10) |
| } |
| |
| // CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithManualStartAndStop' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| // CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+3]]: Test Case 'PerformanceTestCase.test_measuresMetricsWithManualStartAndStop' measured \[Time, seconds\] .* |
| // CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithManualStartAndStop' passed \(\d+\.\d+ seconds\) |
| func test_measuresMetricsWithManualStartAndStop() { |
| measureMetrics([.wallClockTime], automaticallyStartMeasuring: false) { |
| self.startMeasuring() |
| self.stopMeasuring() |
| } |
| } |
| |
| // CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithoutExplicitStop' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| // CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+3]]: Test Case 'PerformanceTestCase.test_measuresMetricsWithoutExplicitStop' measured \[Time, seconds\] .* |
| // CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithoutExplicitStop' passed \(\d+\.\d+ seconds\) |
| func test_measuresMetricsWithoutExplicitStop() { |
| measureMetrics([.wallClockTime], automaticallyStartMeasuring: false) { |
| self.startMeasuring() |
| } |
| } |
| |
| // CHECK: Test Case 'PerformanceTestCase.test_hasWallClockAsDefaultPerformanceMetric' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| // CHECK: Test Case 'PerformanceTestCase.test_hasWallClockAsDefaultPerformanceMetric' passed \(\d+\.\d+ seconds\) |
| func test_hasWallClockAsDefaultPerformanceMetric() { |
| XCTAssertEqual(PerformanceTestCase.defaultPerformanceMetrics, [.wallClockTime]) |
| } |
| |
| // CHECK: Test Case 'PerformanceTestCase.test_printsValuesAfterMeasuring' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| // CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+3]]: Test Case 'PerformanceTestCase.test_printsValuesAfterMeasuring' measured \[Time, seconds\] average: \d+.\d{3}, relative standard deviation: \d+.\d{3}%, values: \[\d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}\], performanceMetricID:org.swift.XCTPerformanceMetric_WallClockTime, maxPercentRelativeStandardDeviation: \d+.\d{3}%, maxStandardDeviation: \d.\d{3} |
| // CHECK: Test Case 'PerformanceTestCase.test_printsValuesAfterMeasuring' passed \(\d+\.\d+ seconds\) |
| func test_printsValuesAfterMeasuring() { |
| measure { |
| Thread.sleep(forTimeInterval: 1) |
| } |
| } |
| |
| // CHECK: Test Case 'PerformanceTestCase.test_abortsMeasurementsAfterTestFailure' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| func test_abortsMeasurementsAfterTestFailure() { |
| var iterationCount = 0 |
| measure { |
| iterationCount += 1 |
| // CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+1]]: error: PerformanceTestCase.test_abortsMeasurementsAfterTestFailure : XCTAssertLessThan failed: \("3"\) is not less than \("3"\) - |
| XCTAssertLessThan(iterationCount, 3) |
| } |
| XCTAssertEqual(iterationCount, 3) |
| } |
| // CHECK: Test Case 'PerformanceTestCase.test_abortsMeasurementsAfterTestFailure' failed \(\d+\.\d+ seconds\) |
| |
| // CHECK: Test Case 'PerformanceTestCase.test_measuresWallClockTimeInBlock' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| func test_measuresWallClockTimeInBlock() { |
| var hasWaited = false |
| // CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+2]]: Test Case 'PerformanceTestCase.test_measuresWallClockTimeInBlock' measured \[Time, seconds\] average: \d+.\d{3}, relative standard deviation: \d+.\d{3}%, values: \[1.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}\], performanceMetricID:org.swift.XCTPerformanceMetric_WallClockTime, maxPercentRelativeStandardDeviation: \d+.\d{3}%, maxStandardDeviation: \d.\d{3} |
| // CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+1]]: error: PerformanceTestCase.test_measuresWallClockTimeInBlock : failed: The relative standard deviation of the measurements is \d+.\d{3}% which is higher than the max allowed of \d+.\d{3}%. |
| measure { |
| if !hasWaited { |
| Thread.sleep(forTimeInterval: 1) |
| hasWaited = true |
| } |
| } |
| } |
| // CHECK: Test Case 'PerformanceTestCase.test_measuresWallClockTimeInBlock' failed \(\d+\.\d+ seconds\) |
| |
| static var allTests = { |
| return [ |
| ("test_measureBlockIteratesTenTimes", test_measureBlockIteratesTenTimes), |
| ("test_measuresMetricsWithAutomaticStartAndStop", test_measuresMetricsWithAutomaticStartAndStop), |
| ("test_measuresMetricsWithManualStartAndStop", test_measuresMetricsWithManualStartAndStop), |
| ("test_measuresMetricsWithoutExplicitStop", test_measuresMetricsWithoutExplicitStop), |
| ("test_hasWallClockAsDefaultPerformanceMetric", test_hasWallClockAsDefaultPerformanceMetric), |
| ("test_printsValuesAfterMeasuring", test_printsValuesAfterMeasuring), |
| ("test_abortsMeasurementsAfterTestFailure", test_abortsMeasurementsAfterTestFailure), |
| ("test_measuresWallClockTimeInBlock", test_measuresWallClockTimeInBlock), |
| ] |
| }() |
| } |
| // CHECK: Test Suite 'PerformanceTestCase' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| // CHECK: \t Executed \d+ tests, with 2 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds |
| |
| XCTMain([testCase(PerformanceTestCase.allTests)]) |
| |
| // CHECK: Test Suite '.*\.xctest' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| // CHECK: \t Executed \d+ tests, with \d failures? \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds |
| // CHECK: Test Suite 'All tests' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+ |
| // CHECK: \t Executed \d+ tests, with \d failures? \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds |
