| #!/usr/bin/env python |
| |
| # ===--- Benchmark_RuntimeLeaksRunner.in ---------------------------------===// |
| # |
| # This source file is part of the Swift.org open source project |
| # |
| # Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors |
| # Licensed under Apache License v2.0 with Runtime Library Exception |
| # |
| # See https://swift.org/LICENSE.txt for license information |
| # See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors |
| # |
| # ===---------------------------------------------------------------------===// |
| |
| import json |
| import os |
| import subprocess |
| import sys |
| |
| sys.path.append("@PATH_TO_DRIVER_LIBRARY@") |
| |
| import perf_test_driver # noqa (E402 module level import not at top of file) |
| |
| # This is a hacked up XFAIL list. It should really be a json file, but it will |
| # work for now. Add in the exact name of the pass to XFAIL. |
| XFAIL_LIST = [ |
| ] |
| |
| # Global Objective-C classes created by various frameworks. We do not care |
| # about these. |
| IGNORABLE_GLOBAL_OBJC_CLASSES = set([ |
| '__NSPlaceholderDate', |
| 'NSCache', |
| '__NSPlaceholderTimeZone', |
| 'NSPlaceholderNumber', |
| 'NSPlaceholderString', |
| '__NSPlaceholderArray', |
| '__NSPlaceholderDictionary', |
| '_NSPlaceholderData', |
| '_NSJSONReader' |
| ]) |
| |
| |
| class LeaksRunnerResult(perf_test_driver.Result): |
| |
| def __init__(self, name, count=None): |
| # True = 1, False = 0. |
| # |
| # TODO: Why are we using booleans to communicate status here? |
| status = count is None or count > 0 |
| perf_test_driver.Result.__init__(self, name, status, "", XFAIL_LIST) |
| self.count = count |
| |
| def get_count(self): |
| if self.count is not None: |
| return str(self.count) |
| return "N/A" |
| |
| def print_data(self, max_test_len): |
| fmt = '{:<%d}{:<10}{:}' % (max_test_len + 5) |
| print(fmt.format(self.get_name(), self.get_result(), |
| self.get_count())) |
| |
| |
| class LeaksRunnerBenchmarkDriver(perf_test_driver.BenchmarkDriver): |
| |
| def __init__(self, binary, xfail_list, num_samples, num_iters): |
| perf_test_driver.BenchmarkDriver.__init__( |
| self, binary, xfail_list, |
| enable_parallel=True) |
| self.num_samples = num_samples |
| self.num_iters = num_iters |
| |
| def print_data_header(self, max_test_len): |
| fmt = '{:<%d}{:<10}{:}' % (max_test_len + 5) |
| print(fmt.format('Name', 'Result', 'RC Delta')) |
| |
| # Propagate any data from this class that is needed for individual |
| # tests. The reason this is needed is to avoid issues with attempting to |
| # access a value in a different process. |
| def prepare_input(self, name): |
| return {'num_samples': self.num_samples, 'num_iters': self.num_iters} |
| |
| def run_test(self, data, num_iters): |
| try: |
| p = subprocess.Popen([ |
| data['path'], "--run-all", |
| "--num-samples={}".format(data['num_samples']), |
| "--num-iters={}".format(num_iters), data['test_name']], |
| stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| error_out = p.communicate()[1].split("\n") |
| result = p.returncode |
| if result is None: |
| raise RuntimeError("Expected one line of output") |
| if result != 0: |
| raise RuntimeError("Process segfaulted") |
| except Exception: |
| sys.stderr.write("Child Process Failed! (%s,%s)\n" % ( |
| data['path'], data['test_name'])) |
| sys.stderr.flush() |
| return None |
| |
| try: |
| # We grab the second line since swift globals get lazily created in |
| # the first iteration. |
| d = json.loads(error_out[1]) |
| d['objc_objects'] = [x for x in d['objc_objects'] |
| if x not in IGNORABLE_GLOBAL_OBJC_CLASSES] |
| d['objc_count'] = len(d['objc_objects']) |
| |
| total_count = d['objc_count'] + d['swift_count'] |
| return total_count |
| except Exception: |
| tmp = (data['path'], data['test_name']) |
| sys.stderr.write("Failed parse output! (%s,%s)\n" % tmp) |
| sys.stderr.flush() |
| return None |
| |
| def process_input(self, data): |
| test_name = '({},{})'.format(data['opt'], data['test_name']) |
| print("Running {}...".format(test_name)) |
| sys.stdout.flush() |
| total_count1 = self.run_test(data, data['num_iters']) |
| if total_count1 is None: |
| return LeaksRunnerResult(test_name) |
| total_count2 = self.run_test(data, data['num_iters'] + 1) |
| if total_count2 is None: |
| return LeaksRunnerResult(test_name) |
| return LeaksRunnerResult(test_name, total_count2 - total_count1) |
| |
| |
| SWIFT_BIN_DIR = os.path.dirname(os.path.abspath(__file__)) |
| |
| |
| def parse_args(): |
| import argparse |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| '-filter', type=str, default=None, |
| help='Filter out any test that does not match the given regex') |
| parser.add_argument('-num-samples', type=int, default=2) |
| parser.add_argument('-num-iters', type=int, default=2) |
| return parser.parse_args() |
| |
| |
| if __name__ == "__main__": |
| args = parse_args() |
| l = LeaksRunnerBenchmarkDriver( |
| SWIFT_BIN_DIR, XFAIL_LIST, args.num_samples, args.num_iters) |
| if l.run(args.filter): |
| sys.exit(0) |
| else: |
| sys.exit(-1) |