| #!/usr/bin/env fuchsia-vendored-python |
| # Copyright 2021 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """ |
| example usage: `./tools/fidl/scripts/syntax_coverage_check.py` |
| """ |
| |
| from collections import namedtuple |
| from pathlib import Path |
| from pprint import pprint |
| import re |
| import os |
| |
| TestCase = namedtuple("TestCase", ["suite", "name"]) |
| |
| IGNORED_FILES = { |
| # syntax agnostic/specific |
| "c_generator_tests.cc", |
| "new_syntax_converter_tests.cc", |
| "flat_ast_tests.cc", |
| "new_syntax_tests.cc", |
| "recursion_detector_tests.cc", |
| "reporter_tests.cc", |
| "type_alias_tests.cc", |
| "virtual_source_tests.cc", |
| "utils_tests.cc", |
| # will be updated along with formatter/linter work |
| "formatter_tests.cc", |
| "json_findings_tests.cc", |
| "lint_findings_tests.cc", |
| "lint_tests.cc", |
| "visitor_unittests.cc", |
| } |
| |
| # success tests that don't have any of the convert macros in them |
| GOOD_TEST_ALLOWLIST = { |
| # these are manually duplicated |
| TestCase("ParsingTests", "GoodAttributeValueHasCorrectContents"), |
| TestCase("ParsingTests", "GoodAttributeValueHasCorrectContentsOld"), |
| TestCase("ParsingTests", "GoodMultilineCommentHasCorrectContents"), |
| TestCase("ParsingTests", "GoodMultilineCommentHasCorrectContentsOld"), |
| TestCase("SpanTests", "GoodParseTest"), |
| TestCase("SpanTests", "GoodParseTestOld"), |
| # these test cases run across both syntaxes without using the macro |
| TestCase("TypesTests", "GoodRootTypesWithNoLibraryInLookup"), |
| TestCase("TypesTests", "GoodRootTypesWithSomeLibraryInLookup"), |
| TestCase("TypesTests", "GoodHandleSubtype"), |
| TestCase("TypesTests", "GoodRights"), |
| # only applies to the new syntax |
| TestCase("ParsingTests", "GoodSingleConstraint"), |
| # only applies to old syntax |
| TestCase("AttributesTests", "GoodAttributeCaseNormalizedOldSyntax"), |
| } |
| |
| # failure tests that aren't a Foo/FooOld pair |
| BAD_TEST_ALLOWLIST = { |
| # old syntax only: no parameter lists in new syntax |
| "BadRecoverToNextParameterInList", |
| # new syntax only: can't specify multiple constraints on array |
| "BadMultipleConstraintsOnArray", |
| # old syntax only: these bugs are fixed in new syntax |
| "BadHandleAlias", |
| "BadBoundsOnRequestType", |
| "BadBoundsOnArray", |
| # new syntax only (constraints) |
| "BadMultipleConstraintsOnPrimitive", |
| "BadNoOptionalOnAliasedPrimitive", |
| "BadMissingConstraintBrackets", |
| "BadEnumMultipleConstraints", |
| "BadTableMultipleConstraints", |
| "BadBitsMultipleConstraints", |
| # this is a pair of old/new syntax tests, they just don't have matching names |
| "BadFinalMemberMissingTypeAndSemicolon", |
| "BadFinalMemberMissingNameAndSemicolon", |
| # new syntax only: can't specify top level request/response type attributes in old |
| "BadRecoverableParamListParsing", |
| # new syntax only (client/server_end, box) |
| "BadParameterizedTypedChannel", |
| "BadTooManyConstraintsTypedChannel", |
| "BadBoxCannotBeNullable", |
| "BadBoxedTypeCannotBeNullable", |
| "BadTypeCannotBeBoxed", |
| # old syntax only: (going to be removed soon) |
| "BadDeprecatedXUnionError", |
| } |
| |
| # TODO: add notes so that they don't need to be entered here |
| ERROR_MISMATCH_ALLOWLIST = { |
| # new syntax does the right thing, old syntax does not |
| "BadBareHandleWithConstraintsThroughAlias", |
| # doesn't apply in new syntax, so we get a syntax error |
| "BadAliasResourceness", |
| "BadProtocolResourceness", |
| "BadConstResourceness", |
| # these return the same underlying error, they just have different names |
| "BadNoAttributeOnUsingNotEventDoc", |
| # type constraint error differences |
| "BadResourceDefinitionMissingRightsPropertyTest", |
| "BadResourceSubtypeNotEnum", |
| "BadResourceDefinitionNonBitsRights", |
| "BadResourceDefinitionMissingSubtypePropertyTest", |
| "BadNonIdentifierSubtype", |
| } |
| |
| FUCHSIA_DIR = os.environ["FUCHSIA_DIR"] |
| |
| WHITE = "\033[1;37m" |
| YELLOW = "\033[1;33m" |
| NC = "\033[0m" |
| |
| ALLOWED_PREFIXES = ["Good", "Bad", "Warn"] |
| |
| |
| def print_color(s, color): |
| print("{}{}{}".format(color, s, NC)) |
| |
| |
| def get_all_test_files(): |
| test_dir = Path(FUCHSIA_DIR) / "tools/fidl/fidlc/tests" |
| for file in test_dir.iterdir(): |
| if file.suffix == ".cc" and file.name not in IGNORED_FILES: |
| yield file |
| |
| |
| if __name__ == "__main__": |
| unlabeled_tests = set() |
| for path in get_all_test_files(): |
| print_color(f"analyzing file: {path.name}", WHITE) |
| with open(path, "r") as f: |
| old_to_errors = {} |
| new_to_errors = {} |
| test_to_notes = {} |
| |
| current = TestCase("", "") |
| is_converted = False |
| errors = [] |
| note = [] |
| in_note = False |
| for line in f: |
| # start of a new test case |
| if line.startswith("TEST("): |
| if current.name.startswith("Good"): |
| if ( |
| not is_converted |
| and current not in GOOD_TEST_ALLOWLIST |
| ): |
| print(f" {current.name}") |
| # pass |
| # TODO: check that tests with imports have 3 copies |
| elif current.name.startswith( |
| "Bad" |
| ) and not current.name.endswith("WithOldDep"): |
| lookup = ( |
| old_to_errors |
| if current.name.endswith("Old") |
| else new_to_errors |
| ) |
| lookup[current.name] = errors |
| if note: |
| test_to_notes[current.name] = " ".join(note) |
| |
| # reset test case state |
| suite = line[line.find("(") + 1 : line.find(",")] |
| name = line[line.find(",") + 2 : line.find(")")] |
| current = TestCase(suite, name) |
| is_converted = False |
| errors = [] |
| note = [] |
| in_note = False |
| if not any( |
| current.name.startswith(p) for p in ALLOWED_PREFIXES |
| ): |
| unlabeled_tests.add(current) |
| continue |
| |
| if ( |
| "ASSERT_COMPILED_AND_CONVERT(" in line |
| or "ASSERT_COMPILED_AND_CONVERT_WITH_DEP(" in line |
| ): |
| assert not current.name.startswith("Bad") |
| is_converted = True |
| continue |
| |
| if "NOTE(https://fxbug.dev/42152439)" in line: |
| assert not current.name.startswith("Good") |
| note.append(line.strip().lstrip("// ")) |
| in_note = True |
| elif in_note: |
| assert not current.name.startswith("Good") |
| if line.lstrip().startswith("// "): |
| note.append(line.strip().lstrip("// ")) |
| else: |
| in_note = False |
| |
| errors.extend(re.findall("fidl::(Err\w+)", line)) |
| errors.extend(re.findall("fidl::(Warn\w+)", line)) |
| |
| # handle the last test |
| if current.name.startswith("Good"): |
| if not is_converted and current not in GOOD_TEST_ALLOWLIST: |
| print(f" {current.name}") |
| # pass |
| elif current.name.startswith("Bad"): |
| lookup = ( |
| old_to_errors |
| if current.name.endswith("Old") |
| else new_to_errors |
| ) |
| lookup[current.name] = errors |
| if note: |
| test_to_notes[current.name] = "\n".join(note) |
| |
| # strip the Old suffix first |
| old_tests = set(t[:-3] for t in old_to_errors.keys()) |
| new_tests = set(t for t in new_to_errors.keys()) |
| for test_name in old_tests | new_tests: |
| if test_name in BAD_TEST_ALLOWLIST: |
| continue |
| if test_name not in old_tests: |
| print(f" missing old: {test_name}Old") |
| continue |
| elif test_name not in new_tests: |
| print(f" missing new: {test_name}") |
| continue |
| |
| old_errors = old_to_errors[test_name + "Old"] |
| new_errors = new_to_errors[test_name] |
| if old_errors != new_errors: |
| if ( |
| test_name not in test_to_notes |
| and test_name not in ERROR_MISMATCH_ALLOWLIST |
| ): |
| # if test_name not in ERROR_MISMATCH_ALLOWLIST: |
| print_color( |
| f" error mismatch for {test_name}: {old_errors} (old) vs {new_errors} (new)", |
| YELLOW, |
| ) |
| # if test_name in test_to_notes: |
| # print('justification:') |
| # print(test_to_notes[test_name]) |
| |
| if unlabeled_tests: |
| print("found unlabeled tests:") |
| pprint(unlabeled_tests) |