blob: cedee0ca6451dfae20614e803096b407f88e5f26 [file] [log] [blame]
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
from __future__ import print_function
import argparse
import os
import sys
import unittest
from contextlib import contextmanager
from io import StringIO
from build_swift import driver_arguments
from build_swift.tests import expected_options as eo
from swift_build_support.swift_build_support import migration
from swift_build_support.swift_build_support.SwiftBuildSupport import (
get_all_preset_names,
get_preset_options,
)
FILE_PATH = os.path.abspath(__file__)
TESTS_PATH = os.path.abspath(os.path.join(FILE_PATH, os.pardir))
BUILD_SWIFT_PATH = os.path.abspath(os.path.join(TESTS_PATH, os.pardir))
UTILS_PATH = os.path.abspath(os.path.join(BUILD_SWIFT_PATH, os.pardir))
BUILD_SCRIPT_IMPL = os.path.join(UTILS_PATH, 'build-script-impl')
PRESETS_FILES = [
os.path.join(UTILS_PATH, 'build-presets.ini'),
]
class ParserError(Exception):
pass
@contextmanager
def redirect_stderr(stream=None):
stream = stream or StringIO()
old_stderr, sys.stderr = sys.stderr, stream
try:
yield stream
finally:
sys.stderr = old_stderr
@contextmanager
def redirect_stdout(stream=None):
stream = stream or StringIO()
old_stdout, sys.stdout = sys.stdout, stream
try:
yield stream
finally:
sys.stdout = old_stdout
def _load_all_presets(presets_files):
preset_names = get_all_preset_names(presets_files)
# Hack to filter out mixins which are not expected to be valid presets
preset_names = [n for n in preset_names if not n.startswith('mixin')]
substitutions = {
'install_destdir': '/tmp/install',
'install_symroot': '/tmp/symroot',
'installable_package': '/tmp/xcode-xyz-root.tar.gz',
}
presets = dict()
for name in preset_names:
try:
# Attempt to parse preset
presets[name] = get_preset_options(substitutions,
presets_files, name)
except SystemExit:
continue
return presets
class TestDriverArgumentParserMeta(type):
"""Metaclass used to dynamically generate test methods for each of the
individual options accepted by the parser and methods to validate all of
the presets.
"""
def __new__(cls, name, bases, attrs):
# Generate tests for each default value
for dest, value in eo.EXPECTED_DEFAULTS.items():
test_name = 'test_default_value_' + dest
attrs[test_name] = cls.generate_default_value_test(dest, value)
# Generate tests for each expected option
for option in eo.EXPECTED_OPTIONS:
test_name = 'test_option_' + option.sanitized_string()
attrs[test_name] = cls.generate_option_test(option)
# Generate tests for each preset
presets = _load_all_presets(PRESETS_FILES)
for name, args in presets.items():
test_name = 'test_preset_' + name
attrs[test_name] = cls.generate_preset_test(name, args)
return super(TestDriverArgumentParserMeta, cls).__new__(
cls, name, bases, attrs)
@classmethod
def generate_default_value_test(cls, dest, default_value):
def test(self):
with self.assertNotRaises(ParserError):
parsed_values = self.parse_default_args([])
parsed_value = getattr(parsed_values, dest)
if default_value.__class__ is str:
parsed_value = str(parsed_value)
self.assertEqual(default_value, parsed_value,
'Invalid default value for "{}": {} != {}'
.format(dest, default_value, parsed_value))
return test
@classmethod
def _generate_help_option_test(cls, option):
def test(self):
with redirect_stdout() as output, self.assertRaises(ParserError):
self.parse_args([option.option_string])
self.assertNotEmpty(output)
return test
@classmethod
def _generate_set_option_test(cls, option):
def test(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_args([option.option_string])
self.assertEqual(getattr(namespace, option.dest), option.value)
with self.assertRaises(ParserError):
self.parse_args([option.option_string, 'foo'])
return test
@classmethod
def _generate_set_true_option_test(cls, option):
def test(self):
with self.assertNotRaises(ParserError):
# TODO: Move to unit-tests for the action class
namespace = self.parse_args([])
self.assertFalse(getattr(namespace, option.dest))
namespace = self.parse_args([option.option_string])
self.assertTrue(getattr(namespace, option.dest))
return test
@classmethod
def _generate_set_false_option_test(cls, option):
def test(self):
with self.assertNotRaises(ParserError):
# TODO: Move to unit-tests for the action class
namespace = self.parse_args([])
self.assertTrue(getattr(namespace, option.dest))
namespace = self.parse_args([option.option_string])
self.assertFalse(getattr(namespace, option.dest))
return test
@classmethod
def _generate_enable_option_test(cls, option):
def test(self):
with self.assertNotRaises(ParserError):
# TODO: Move to unit-tests for the action class
# Test parsing True values
self.parse_args([option.option_string, '1'])
self.parse_args([option.option_string, 'true'])
self.parse_args([option.option_string, 'True'])
self.parse_args([option.option_string, 'TRUE'])
# TODO: Move to unit-tests for the action class
# Test parsing False values
self.parse_args([option.option_string, '0'])
self.parse_args([option.option_string, 'false'])
self.parse_args([option.option_string, 'False'])
self.parse_args([option.option_string, 'FALSE'])
# TODO: Move to unit-tests for the action class
# Test default value
namespace = self.parse_args([option.option_string])
self.assertTrue(getattr(namespace, option.dest))
# Test setting value to True
namespace = self.parse_args([option.option_string, 'True'])
self.assertTrue(getattr(namespace, option.dest))
# Test setting value to False
namespace = self.parse_args([option.option_string, 'False'])
self.assertFalse(getattr(namespace, option.dest))
return test
@classmethod
def _generate_disable_option_test(cls, option):
def test(self):
with self.assertNotRaises(ParserError):
# TODO: Move to unit-tests for the action class
# Test parsing True values
self.parse_args([option.option_string, '1'])
self.parse_args([option.option_string, 'true'])
self.parse_args([option.option_string, 'True'])
self.parse_args([option.option_string, 'TRUE'])
# TODO: Move to unit-tests for the action class
# Test parsing False values
self.parse_args([option.option_string, '0'])
self.parse_args([option.option_string, 'false'])
self.parse_args([option.option_string, 'False'])
self.parse_args([option.option_string, 'FALSE'])
# TODO: Move to unit-tests for the action class
# Test default value
namespace = self.parse_args([option.option_string])
self.assertFalse(getattr(namespace, option.dest))
# Test setting value to True resulting in False
namespace = self.parse_args([option.option_string, 'True'])
self.assertFalse(getattr(namespace, option.dest))
# Test setting value to False resulting in True
namespace = self.parse_args([option.option_string, 'False'])
self.assertTrue(getattr(namespace, option.dest))
return test
@classmethod
def _generate_choices_option_test(cls, option):
def test(self):
with self.assertNotRaises(ParserError):
for choice in option.choices:
namespace = self.parse_args(
[option.option_string, str(choice)])
self.assertEqual(getattr(namespace, option.dest), choice)
with self.assertRaises(ParserError):
self.parse_args([option.option_string, 'INVALID'])
return test
@classmethod
def _generate_int_option_test(cls, option):
def test(self):
with self.assertNotRaises(ParserError):
for i in [0, 1, 42]:
namespace = self.parse_args([option.option_string, str(i)])
self.assertEqual(int(getattr(namespace, option.dest)), i)
# FIXME: int-type options should not accept non-int strings
# with self.assertRaises(ParserError):
# self.parse_args([option.option_string, str(0.0)])
# self.parse_args([option.option_string, str(1.0)])
# self.parse_args([option.option_string, str(3.14)])
# self.parse_args([option.option_string, 'NaN'])
return test
@classmethod
def _generate_str_option_test(cls, option):
def test(self):
with self.assertNotRaises(ParserError):
self.parse_args([option.option_string, 'foo'])
return test
@classmethod
def _generate_path_option_test(cls, option):
def test(self):
with self.assertNotRaises(ParserError):
self.parse_args([option.option_string, sys.executable])
# FIXME: path-type options should not accept non-path inputs
# with self.assertRaises(ParserError):
# self.parse_args([option.option_string, 'foo'])
return test
@classmethod
def _generate_append_option_test(cls, option):
def test(self):
with self.assertNotRaises(ParserError):
# Range size is arbitrary, just needs to be more than once
for i in range(1, 4):
namespace = self.parse_args(
[option.option_string, 'ARG'] * i)
self.assertEqual(getattr(namespace, option.dest),
['ARG'] * i)
return test
@classmethod
def _generate_unsupported_option_test(cls, option):
def test(self):
with self.assertRaises(ParserError):
self.parse_args([option.option_string])
return test
@classmethod
def generate_option_test(cls, option):
generate_test_funcs = {
eo.HelpOption: cls._generate_help_option_test,
eo.SetOption: cls._generate_set_option_test,
eo.SetTrueOption: cls._generate_set_true_option_test,
eo.SetFalseOption: cls._generate_set_false_option_test,
eo.EnableOption: cls._generate_enable_option_test,
eo.DisableOption: cls._generate_disable_option_test,
eo.ChoicesOption: cls._generate_choices_option_test,
eo.IntOption: cls._generate_int_option_test,
eo.StrOption: cls._generate_str_option_test,
eo.PathOption: cls._generate_path_option_test,
eo.AppendOption: cls._generate_append_option_test,
eo.UnsupportedOption: cls._generate_unsupported_option_test,
# IgnoreOptions should be manually tested
eo.IgnoreOption: lambda self: None,
}
test_func = generate_test_funcs.get(option.__class__, None)
if test_func is not None:
return test_func(option)
# Catch-all meaningless test
return lambda self: \
self.fail('unexpected option "{}"'.format(option.option_string))
@classmethod
def generate_preset_test(cls, preset_name, preset_args):
def test(self):
try:
self.parse_default_args(preset_args, check_impl_args=True)
except ParserError as e:
self.fail('failed to parse preset "{}": {}'.format(
preset_name, e))
return test
class TestDriverArgumentParser(unittest.TestCase):
__metaclass__ = TestDriverArgumentParserMeta
@contextmanager
def _quiet_output(self):
with open(os.devnull, 'w') as devnull:
with redirect_stderr(devnull), redirect_stdout(devnull):
yield
def _parse_args(self, args):
try:
return migration.parse_args(self.parser, args)
except (SystemExit, ValueError) as e:
raise ParserError('failed to parse arguments: ' +
str(args), e)
def _check_impl_args(self, namespace):
assert hasattr(namespace, 'build_script_impl_args')
try:
migration.check_impl_args(BUILD_SCRIPT_IMPL,
namespace.build_script_impl_args)
except (SystemExit, ValueError) as e:
raise ParserError('failed to parse impl arguments: ' +
str(namespace.build_script_impl_args), e)
def parse_args(self, args, namespace=None):
if namespace is None:
namespace = argparse.Namespace()
with self._quiet_output():
try:
namespace, unknown_args =\
super(self.parser.__class__, self.parser)\
.parse_known_args(args, namespace)
except (SystemExit, argparse.ArgumentError) as e:
raise ParserError('failed to parse arguments: ' + str(args), e)
if unknown_args:
raise ParserError('unknown arguments: ' + str(unknown_args))
return namespace
def parse_default_args(self, args, check_impl_args=False):
with self._quiet_output():
namespace = self._parse_args(args)
if check_impl_args:
self._check_impl_args(namespace)
return namespace
@contextmanager
def assertNotRaises(self, exception):
assert issubclass(exception, BaseException)
try:
yield
except exception as e:
self.fail(str(e))
def setUp(self):
self.parser = driver_arguments.create_argument_parser()
# -------------------------------------------------------------------------
def test_expected_options_exhaustive(self):
"""Test that we are exhaustively testing all options accepted by the
parser. If this test if failing then the parser accepts more options
than currently being tested, meaning the EXPECTED_OPTIONS list in
build_swift/tests/expected_options.py should be updated to include
the missing options.
"""
expected_options = {o.option_string for o in eo.EXPECTED_OPTIONS}
# aggregate and flatten the options_strings accepted by the parser
actual_options = [a.option_strings for a in self.parser._actions]
actual_options = set(sum(actual_options, []))
diff = actual_options - expected_options
if len(diff) > 0:
self.fail('non-exhaustive expected options, missing: {}'
.format(diff))
def test_expected_options_have_default_values(self):
"""Test that all the options in EXPECTED_OPTIONS have an associated
default value.
"""
skip_option_classes = [
eo.HelpOption,
eo.IgnoreOption,
eo.UnsupportedOption,
]
missing_defaults = set()
for option in eo.EXPECTED_OPTIONS:
if option.__class__ in skip_option_classes:
continue
if option.dest not in eo.EXPECTED_DEFAULTS:
missing_defaults.add(option.dest)
if len(missing_defaults) > 0:
self.fail('non-exhaustive default values for options, missing: {}'
.format(missing_defaults))
# -------------------------------------------------------------------------
# Manual option tests
def test_option_clang_compiler_version(self):
option_string = '--clang-compiler-version'
with self.assertNotRaises(ParserError):
self.parse_default_args([option_string, '5.0.0'])
self.parse_default_args([option_string, '5.0.1'])
self.parse_default_args([option_string, '5.0.0.1'])
with self.assertRaises(ParserError):
self.parse_default_args([option_string, '1'])
self.parse_default_args([option_string, '1.2'])
self.parse_default_args([option_string, '0.0.0.0.1'])
def test_option_clang_user_visible_version(self):
option_string = '--clang-user-visible-version'
with self.assertNotRaises(ParserError):
self.parse_default_args([option_string, '5.0.0'])
self.parse_default_args([option_string, '5.0.1'])
self.parse_default_args([option_string, '5.0.0.1'])
with self.assertRaises(ParserError):
self.parse_default_args([option_string, '1'])
self.parse_default_args([option_string, '1.2'])
self.parse_default_args([option_string, '0.0.0.0.1'])
def test_option_swift_compiler_version(self):
option_string = '--swift-compiler-version'
with self.assertNotRaises(ParserError):
self.parse_default_args([option_string, '4.1'])
self.parse_default_args([option_string, '4.0.1'])
self.parse_default_args([option_string, '200.99.1'])
with self.assertRaises(ParserError):
self.parse_default_args([option_string, '1'])
self.parse_default_args([option_string, '0.0.0.1'])
def test_option_swift_user_visible_version(self):
option_string = '--swift-user-visible-version'
with self.assertNotRaises(ParserError):
self.parse_default_args([option_string, '4.1'])
self.parse_default_args([option_string, '4.0.1'])
self.parse_default_args([option_string, '200.99.1'])
with self.assertRaises(ParserError):
self.parse_default_args([option_string, '1'])
self.parse_default_args([option_string, '0.0.0.1'])
def test_option_I(self):
with self.assertRaises(ValueError):
self.parse_default_args(['-I'])
def test_option_ios_all(self):
with self.assertRaises(ValueError):
self.parse_default_args(['--ios-all'])
def test_option_tvos_all(self):
with self.assertRaises(ValueError):
self.parse_default_args(['--tvos-all'])
def test_option_watchos_all(self):
with self.assertRaises(ValueError):
self.parse_default_args(['--watchos-all'])
# -------------------------------------------------------------------------
# Implied defaults tests
def test_implied_defaults_assertions(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--assertions'])
self.assertTrue(namespace.cmark_assertions)
self.assertTrue(namespace.llvm_assertions)
self.assertTrue(namespace.swift_assertions)
self.assertTrue(namespace.swift_stdlib_assertions)
def test_implied_defaults_cmark_build_variant(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--debug-cmark'])
self.assertTrue(namespace.build_cmark)
def test_implied_defaults_lldb_build_variant(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--debug-lldb'])
self.assertTrue(namespace.build_lldb)
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--lldb-assertions'])
self.assertTrue(namespace.build_lldb)
def test_implied_defaults_build_variant(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--debug'])
self.assertEquals(namespace.cmark_build_variant, 'Debug')
self.assertEquals(namespace.foundation_build_variant, 'Debug')
self.assertEquals(namespace.libdispatch_build_variant, 'Debug')
self.assertEquals(namespace.libicu_build_variant, 'Debug')
self.assertEquals(namespace.lldb_build_variant, 'Debug')
self.assertEquals(namespace.llvm_build_variant, 'Debug')
self.assertEquals(namespace.swift_build_variant, 'Debug')
self.assertEquals(namespace.swift_stdlib_build_variant, 'Debug')
def test_implied_defaults_skip_build(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--skip-build'])
self.assertFalse(namespace.build_benchmarks)
self.assertFalse(namespace.build_linux)
self.assertFalse(namespace.build_android)
self.assertFalse(namespace.build_freebsd)
self.assertFalse(namespace.build_cygwin)
self.assertFalse(namespace.build_osx)
self.assertFalse(namespace.build_ios)
self.assertFalse(namespace.build_tvos)
self.assertFalse(namespace.build_watchos)
self.assertFalse(namespace.build_foundation)
self.assertFalse(namespace.build_libdispatch)
self.assertFalse(namespace.build_libicu)
self.assertFalse(namespace.build_lldb)
self.assertFalse(namespace.build_llbuild)
self.assertFalse(namespace.build_playgroundlogger)
self.assertFalse(namespace.build_playgroundsupport)
self.assertFalse(namespace.build_swiftpm)
self.assertFalse(namespace.build_xctest)
def test_implied_defaults_skip_build_ios(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--skip-build-ios'])
self.assertFalse(namespace.build_ios_device)
self.assertFalse(namespace.build_ios_simulator)
# Also implies that the tests should be skipped
self.assertFalse(namespace.test_ios_host)
self.assertFalse(namespace.test_ios_simulator)
def test_implied_defaults_skip_build_tvos(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--skip-build-tvos'])
self.assertFalse(namespace.build_tvos_device)
self.assertFalse(namespace.build_tvos_simulator)
# Also implies that the tests should be skipped
self.assertFalse(namespace.test_tvos_host)
self.assertFalse(namespace.test_tvos_simulator)
def test_implied_defaults_skip_build_watchos(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--skip-build-watchos'])
self.assertFalse(namespace.build_watchos_device)
self.assertFalse(namespace.build_watchos_simulator)
# Also implies that the tests should be skipped
self.assertFalse(namespace.test_watchos_host)
self.assertFalse(namespace.test_watchos_simulator)
def test_implied_defaults_validation_test(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--validation-test'])
self.assertTrue(namespace.test)
def test_implied_defaults_test_optimized(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--test-optimized'])
self.assertTrue(namespace.test)
def test_implied_defaults_test_optimize_for_size(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--test-optimize-for-size'])
self.assertTrue(namespace.test)
def test_implied_defaults_skip_all_tests(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args([
'--test', '0',
'--validation-test', '0',
'--long-test', '0',
])
self.assertFalse(namespace.test_linux)
self.assertFalse(namespace.test_freebsd)
self.assertFalse(namespace.test_cygwin)
self.assertFalse(namespace.test_osx)
self.assertFalse(namespace.test_ios)
self.assertFalse(namespace.test_tvos)
self.assertFalse(namespace.test_watchos)
def test_implied_defaults_skip_test_ios(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--skip-test-ios'])
self.assertFalse(namespace.test_ios_host)
self.assertFalse(namespace.test_ios_simulator)
def test_implied_defaults_skip_test_tvos(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--skip-test-tvos'])
self.assertFalse(namespace.test_tvos_host)
self.assertFalse(namespace.test_tvos_simulator)
def test_implied_defaults_skip_test_watchos(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--skip-test-watchos'])
self.assertFalse(namespace.test_watchos_host)
self.assertFalse(namespace.test_watchos_simulator)
def test_implied_defaults_skip_build_android(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--android', '0'])
self.assertFalse(namespace.test_android_host)
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--skip-build-android'])
self.assertFalse(namespace.test_android_host)
def test_implied_defaults_host_test(self):
with self.assertNotRaises(ParserError):
namespace = self.parse_default_args(['--host-test', '0'])
self.assertFalse(namespace.test_ios_host)
self.assertFalse(namespace.test_tvos_host)
self.assertFalse(namespace.test_watchos_host)
self.assertFalse(namespace.test_android_host)
if __name__ == '__main__':
unittest.main()