tools/fidl/scripts/syntax_coverage_check.py

#!/usr/bin/env fuchsia-vendored-python
# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
example usage: `./tools/fidl/scripts/syntax_coverage_check.py`
"""

from collections import namedtuple
from pathlib import Path
from pprint import pprint
import re
import os

TestCase = namedtuple("TestCase", ["suite", "name"])

IGNORED_FILES = {
    # syntax agnostic/specific
    "c_generator_tests.cc",
    "new_syntax_converter_tests.cc",
    "flat_ast_tests.cc",
    "new_syntax_tests.cc",
    "recursion_detector_tests.cc",
    "reporter_tests.cc",
    "type_alias_tests.cc",
    "virtual_source_tests.cc",
    "utils_tests.cc",
    # will be updated along with formatter/linter work
    "formatter_tests.cc",
    "json_findings_tests.cc",
    "lint_findings_tests.cc",
    "lint_tests.cc",
    "visitor_unittests.cc",
}

# success tests that don't have any of the convert macros in them
GOOD_TEST_ALLOWLIST = {
    # these are manually duplicated
    TestCase("ParsingTests", "GoodAttributeValueHasCorrectContents"),
    TestCase("ParsingTests", "GoodAttributeValueHasCorrectContentsOld"),
    TestCase("ParsingTests", "GoodMultilineCommentHasCorrectContents"),
    TestCase("ParsingTests", "GoodMultilineCommentHasCorrectContentsOld"),
    TestCase("SpanTests", "GoodParseTest"),
    TestCase("SpanTests", "GoodParseTestOld"),
    # these test cases run across both syntaxes without using the macro
    TestCase("TypesTests", "GoodRootTypesWithNoLibraryInLookup"),
    TestCase("TypesTests", "GoodRootTypesWithSomeLibraryInLookup"),
    TestCase("TypesTests", "GoodHandleSubtype"),
    TestCase("TypesTests", "GoodRights"),
    # only applies to the new syntax
    TestCase("ParsingTests", "GoodSingleConstraint"),
    # only applies to old syntax
    TestCase("AttributesTests", "GoodAttributeCaseNormalizedOldSyntax"),
}

# failure tests that aren't a Foo/FooOld pair
BAD_TEST_ALLOWLIST = {
    # old syntax only: no parameter lists in new syntax
    "BadRecoverToNextParameterInList",
    # new syntax only: can't specify multiple constraints on array
    "BadMultipleConstraintsOnArray",
    # old syntax only: these bugs are fixed in new syntax
    "BadHandleAlias",
    "BadBoundsOnRequestType",
    "BadBoundsOnArray",
    # new syntax only (constraints)
    "BadMultipleConstraintsOnPrimitive",
    "BadNoOptionalOnAliasedPrimitive",
    "BadMissingConstraintBrackets",
    "BadEnumMultipleConstraints",
    "BadTableMultipleConstraints",
    "BadBitsMultipleConstraints",
    # this is a pair of old/new syntax tests, they just don't have matching names
    "BadFinalMemberMissingTypeAndSemicolon",
    "BadFinalMemberMissingNameAndSemicolon",
    # new syntax only: can't specify top level request/response type attributes in old
    "BadRecoverableParamListParsing",
    # new syntax only (client/server_end, box)
    "BadParameterizedTypedChannel",
    "BadTooManyConstraintsTypedChannel",
    "BadBoxCannotBeNullable",
    "BadBoxedTypeCannotBeNullable",
    "BadTypeCannotBeBoxed",
    # old syntax only: (going to be removed soon)
    "BadDeprecatedXUnionError",
}

# TODO: add notes so that they don't need to be entered here
ERROR_MISMATCH_ALLOWLIST = {
    # new syntax does the right thing, old syntax does not
    "BadBareHandleWithConstraintsThroughAlias",
    # doesn't apply in new syntax, so we get a syntax error
    "BadAliasResourceness",
    "BadProtocolResourceness",
    "BadConstResourceness",
    # these return the same underlying error, they just have different names
    "BadNoAttributeOnUsingNotEventDoc",
    # type constraint error differences
    "BadResourceDefinitionMissingRightsPropertyTest",
    "BadResourceSubtypeNotEnum",
    "BadResourceDefinitionNonBitsRights",
    "BadResourceDefinitionMissingSubtypePropertyTest",
    "BadNonIdentifierSubtype",
}

FUCHSIA_DIR = os.environ["FUCHSIA_DIR"]

WHITE = "\033[1;37m"
YELLOW = "\033[1;33m"
NC = "\033[0m"

ALLOWED_PREFIXES = ["Good", "Bad", "Warn"]


def print_color(s, color):
    print("{}{}{}".format(color, s, NC))


def get_all_test_files():
    test_dir = Path(FUCHSIA_DIR) / "tools/fidl/fidlc/tests"
    for file in test_dir.iterdir():
        if file.suffix == ".cc" and file.name not in IGNORED_FILES:
            yield file


if __name__ == "__main__":
    unlabeled_tests = set()
    for path in get_all_test_files():
        print_color(f"analyzing file: {path.name}", WHITE)
        with open(path, "r") as f:
            old_to_errors = {}
            new_to_errors = {}
            test_to_notes = {}

            current = TestCase("", "")
            is_converted = False
            errors = []
            note = []
            in_note = False
            for line in f:
                # start of a new test case
                if line.startswith("TEST("):
                    if current.name.startswith("Good"):
                        if (
                            not is_converted
                            and current not in GOOD_TEST_ALLOWLIST
                        ):
                            print(f"  {current.name}")
                            # pass
                    # TODO: check that tests with imports have 3 copies
                    elif current.name.startswith(
                        "Bad"
                    ) and not current.name.endswith("WithOldDep"):
                        lookup = (
                            old_to_errors
                            if current.name.endswith("Old")
                            else new_to_errors
                        )
                        lookup[current.name] = errors
                        if note:
                            test_to_notes[current.name] = " ".join(note)

                    # reset test case state
                    suite = line[line.find("(") + 1 : line.find(",")]
                    name = line[line.find(",") + 2 : line.find(")")]
                    current = TestCase(suite, name)
                    is_converted = False
                    errors = []
                    note = []
                    in_note = False
                    if not any(
                        current.name.startswith(p) for p in ALLOWED_PREFIXES
                    ):
                        unlabeled_tests.add(current)
                    continue

                if (
                    "ASSERT_COMPILED_AND_CONVERT(" in line
                    or "ASSERT_COMPILED_AND_CONVERT_WITH_DEP(" in line
                ):
                    assert not current.name.startswith("Bad")
                    is_converted = True
                    continue

                if "NOTE(https://fxbug.dev/42152439)" in line:
                    assert not current.name.startswith("Good")
                    note.append(line.strip().lstrip("// "))
                    in_note = True
                elif in_note:
                    assert not current.name.startswith("Good")
                    if line.lstrip().startswith("// "):
                        note.append(line.strip().lstrip("// "))
                    else:
                        in_note = False

                errors.extend(re.findall("fidl::(Err\w+)", line))
                errors.extend(re.findall("fidl::(Warn\w+)", line))

            # handle the last test
            if current.name.startswith("Good"):
                if not is_converted and current not in GOOD_TEST_ALLOWLIST:
                    print(f"  {current.name}")
                    # pass
            elif current.name.startswith("Bad"):
                lookup = (
                    old_to_errors
                    if current.name.endswith("Old")
                    else new_to_errors
                )
                lookup[current.name] = errors
                if note:
                    test_to_notes[current.name] = "\n".join(note)

        # strip the Old suffix first
        old_tests = set(t[:-3] for t in old_to_errors.keys())
        new_tests = set(t for t in new_to_errors.keys())
        for test_name in old_tests | new_tests:
            if test_name in BAD_TEST_ALLOWLIST:
                continue
            if test_name not in old_tests:
                print(f" missing old: {test_name}Old")
                continue
            elif test_name not in new_tests:
                print(f" missing new: {test_name}")
                continue

            old_errors = old_to_errors[test_name + "Old"]
            new_errors = new_to_errors[test_name]
            if old_errors != new_errors:
                if (
                    test_name not in test_to_notes
                    and test_name not in ERROR_MISMATCH_ALLOWLIST
                ):
                    # if test_name not in ERROR_MISMATCH_ALLOWLIST:
                    print_color(
                        f"  error mismatch for {test_name}: {old_errors} (old) vs {new_errors} (new)",
                        YELLOW,
                    )
                # if test_name in test_to_notes:
                #     print('justification:')
                #     print(test_to_notes[test_name])

    if unlabeled_tests:
        print("found unlabeled tests:")
        pprint(unlabeled_tests)