tools/fidl/scripts/syntax_coverage_check.py

#!/usr/bin/env python3.8
# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
example usage: `./tools/fidl/scripts/syntax_coverage_check.py`
"""

from collections import namedtuple
from pathlib import Path
from pprint import pprint
import re
import os

TestCase = namedtuple('TestCase', ['suite', 'name'])

IGNORED_FILES = {
    # syntax agnostic/specific
    'c_generator_tests.cc',
    'new_syntax_converter_tests.cc',
    'flat_ast_tests.cc',
    'new_syntax_tests.cc',
    'recursion_detector_tests.cc',
    'reporter_tests.cc',
    'type_alias_tests.cc',
    'virtual_source_tests.cc',
    'utils_tests.cc',
    # will be updated along with formatter/linter work
    'formatter_tests.cc',
    'json_findings_tests.cc',
    'lint_findings_tests.cc',
    'lint_tests.cc',
    'visitor_unittests.cc',
}

# success tests that don't have any of the convert macros in them
GOOD_TEST_ALLOWLIST = {
    # these are manually duplicated
    TestCase('ParsingTests', 'GoodAttributeValueHasCorrectContents'),
    TestCase('ParsingTests', 'GoodAttributeValueHasCorrectContentsOld'),
    TestCase('ParsingTests', 'GoodMultilineCommentHasCorrectContents'),
    TestCase('ParsingTests', 'GoodMultilineCommentHasCorrectContentsOld'),
    TestCase('SpanTests', 'GoodParseTest'),
    TestCase('SpanTests', 'GoodParseTestOld'),
    # these test cases run across both syntaxes without using the macro
    TestCase('TypesTests', 'GoodRootTypesWithNoLibraryInLookup'),
    TestCase('TypesTests', 'GoodRootTypesWithSomeLibraryInLookup'),
    TestCase('TypesTests', 'GoodHandleSubtype'),
    TestCase('TypesTests', 'GoodRights'),
    # only applies to the new syntax
    TestCase('ParsingTests', 'GoodSingleConstraint'),
    # only applies to old syntax
    TestCase('AttributesTests', 'GoodAttributeCaseNormalizedOldSyntax'),
}

# failure tests that aren't a Foo/FooOld pair
BAD_TEST_ALLOWLIST = {
    # old syntax only: no parameter lists in new syntax
    'BadRecoverToNextParameterInList',
    # new syntax only: can't specify multiple constraints on array
    'BadMultipleConstraintsOnArray',
    # old syntax only: these bugs are fixed in new syntax
    'BadHandleAlias',
    'BadBoundsOnRequestType',
    'BadBoundsOnArray',
    # new syntax only (constraints)
    'BadMultipleConstraintsOnPrimitive',
    'BadNoOptionalOnAliasedPrimitive',
    'BadMissingConstraintBrackets',
    'BadEnumMultipleConstraints',
    'BadTableMultipleConstraints',
    'BadBitsMultipleConstraints',
    # this is a pair of old/new syntax tests, they just don't have matching names
    'BadFinalMemberMissingTypeAndSemicolon',
    'BadFinalMemberMissingNameAndSemicolon',
    # new syntax only: can't specify top level request/response type attributes in old
    'BadRecoverableParamListParsing',
    # new syntax only (client/server_end, box)
    'BadParameterizedTypedChannel',
    'BadTooManyConstraintsTypedChannel',
    'BadBoxCannotBeNullable',
    'BadBoxedTypeCannotBeNullable',
    'BadTypeCannotBeBoxed',
    # old syntax only: (going to be removed soon)
    'BadDeprecatedXUnionError',
}

# TODO: add notes so that they don't need to be entered here
ERROR_MISMATCH_ALLOWLIST = {
    # new syntax does the right thing, old syntax does not
    'BadBareHandleWithConstraintsThroughAlias',
    # doesn't apply in new syntax, so we get a syntax error
    'BadAliasResourceness',
    'BadProtocolResourceness',
    'BadConstResourceness',
    # these return the same underlying error, they just have different names
    'BadNoAttributeOnUsingNotEventDoc',
    # type constraint error differences
    'BadResourceDefinitionMissingRightsPropertyTest',
    'BadResourceSubtypeNotEnum',
    'BadResourceDefinitionNonBitsRights',
    'BadResourceDefinitionMissingSubtypePropertyTest',
    'BadNonIdentifierSubtype',
}

FUCHSIA_DIR = os.environ['FUCHSIA_DIR']

WHITE = '\033[1;37m'
YELLOW = '\033[1;33m'
NC = '\033[0m'

ALLOWED_PREFIXES = ['Good', 'Bad', 'Warn']


def print_color(s, color):
    print('{}{}{}'.format(color, s, NC))


def get_all_test_files():
    test_dir = Path(FUCHSIA_DIR) / 'tools/fidl/fidlc/tests'
    for file in test_dir.iterdir():
        if file.suffix == '.cc' and file.name not in IGNORED_FILES:
            yield file


if __name__ == '__main__':
    unlabeled_tests = set()
    for path in get_all_test_files():
        print_color(f'analyzing file: {path.name}', WHITE)
        with open(path, 'r') as f:
            old_to_errors = {}
            new_to_errors = {}
            test_to_notes = {}

            current = TestCase('', '')
            is_converted = False
            errors = []
            note = []
            in_note = False
            for line in f:
                # start of a new test case
                if line.startswith('TEST('):
                    if current.name.startswith('Good'):
                        if not is_converted and current not in GOOD_TEST_ALLOWLIST:
                            print(f'  {current.name}')
                            # pass
                    # TODO: check that tests with imports have 3 copies
                    elif current.name.startswith(
                            'Bad') and not current.name.endswith('WithOldDep'):
                        lookup = old_to_errors if current.name.endswith(
                            'Old') else new_to_errors
                        lookup[current.name] = errors
                        if note:
                            test_to_notes[current.name] = ' '.join(note)

                    # reset test case state
                    suite = line[line.find('(') + 1:line.find(',')]
                    name = line[line.find(',') + 2:line.find(')')]
                    current = TestCase(suite, name)
                    is_converted = False
                    errors = []
                    note = []
                    in_note = False
                    if not any(current.name.startswith(p)
                               for p in ALLOWED_PREFIXES):
                        unlabeled_tests.add(current)
                    continue

                if 'ASSERT_COMPILED_AND_CONVERT(' in line or 'ASSERT_COMPILED_AND_CONVERT_WITH_DEP(' in line:
                    assert not current.name.startswith('Bad')
                    is_converted = True
                    continue

                if 'NOTE(fxbug.dev/72924)' in line:
                    assert not current.name.startswith('Good')
                    note.append(line.strip().lstrip('// '))
                    in_note = True
                elif in_note:
                    assert not current.name.startswith('Good')
                    if line.lstrip().startswith('// '):
                        note.append(line.strip().lstrip('// '))
                    else:
                        in_note = False

                errors.extend(re.findall('fidl::(Err\w+)', line))
                errors.extend(re.findall('fidl::(Warn\w+)', line))

            # handle the last test
            if current.name.startswith('Good'):
                if not is_converted and current not in GOOD_TEST_ALLOWLIST:
                    print(f'  {current.name}')
                    # pass
            elif current.name.startswith('Bad'):
                lookup = old_to_errors if current.name.endswith(
                    'Old') else new_to_errors
                lookup[current.name] = errors
                if note:
                    test_to_notes[current.name] = '\n'.join(note)

        # strip the Old suffix first
        old_tests = set(t[:-3] for t in old_to_errors.keys())
        new_tests = set(t for t in new_to_errors.keys())
        for test_name in old_tests | new_tests:
            if test_name in BAD_TEST_ALLOWLIST:
                continue
            if test_name not in old_tests:
                print(f' missing old: {test_name}Old')
                continue
            elif test_name not in new_tests:
                print(f' missing new: {test_name}')
                continue

            old_errors = old_to_errors[test_name + 'Old']
            new_errors = new_to_errors[test_name]
            if old_errors != new_errors:
                if test_name not in test_to_notes and test_name not in ERROR_MISMATCH_ALLOWLIST:
                    # if test_name not in ERROR_MISMATCH_ALLOWLIST:
                    print_color(
                        f'  error mismatch for {test_name}: {old_errors} (old) vs {new_errors} (new)',
                        YELLOW)
                # if test_name in test_to_notes:
                #     print('justification:')
                #     print(test_to_notes[test_name])

    if unlabeled_tests:
        print('found unlabeled tests:')
        pprint(unlabeled_tests)