recipes/disabled_tests.resources/find_disabled_tests.py

#!/usr/bin/env python3
# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Find disabled tests.

This script assumes that it's run in the root directory of a Fuchsia
checkout.
"""
# pylint: disable=consider-using-with,unspecified-encoding

import datetime
import json
import os
import sys
import tempfile
import unittest
import re

# Files matching these types are scraped for various flags and patterns associated with
# disabled tests.
RUST_EXT = ".rs"
DART_EXT = ".dart"
BUILD_EXT = ".gn"
C_EXT = ".cc"

# Mapping language to method of disablement.
# Lines with flags that match these are considered disabled. See
# https://fuchsia.dev/fuchsia-src/development/testing/faq#disable-test for more details on
# language-specific approaches
DISABLED_FLAG_MAP = {
    C_EXT: [
        "DISABLED_",
        ".disabled = true",
    ],
    RUST_EXT: [
        "#[test]\n#[ignore]",
    ],
}

# Additionally, tests can be disabled by being commented out. This regex searches files for comments
# that reference disabling or enabling tests or flakes. This is a hacky approach, but catches many
# disabled tests.
TODO_ENABLE = re.compile(r"^(\/\/|\#).*(TODO).*(enable|disable).*(flake|test).*")

# This regex intends to capture a bug, if one is present in a comment caught by the TODO_ENABLE regex.
FXB_REGEX = re.compile(r"fxb.*\/(\d+)")


def main():
    args = sys.argv[1:]
    tmp_json = args[0]
    curpath = os.path.abspath(os.path.curdir)
    disabled_tests = []

    exclude = set(["third_party"])
    # Search for files matching select file extensions
    for dirpath, dirs, files in os.walk(curpath):
        dirs[:] = [d for d in dirs if d not in exclude]
        for filename in files:
            fname = os.path.join(dirpath, filename)
            file_ext = os.path.splitext(filename)[1]
            if file_ext in (RUST_EXT, DART_EXT, BUILD_EXT, C_EXT):
                find_disabled_tests(fname, disabled_tests, curpath, file_ext)

    with open(tmp_json, "w") as f:
        json.dump(disabled_tests, f, indent=1, separators=(",", ": "))


# Identify disabled tests in a file by searching for either a lanague-specific flag or a comment matching
# the TODO_ENABLE regex.
def find_disabled_tests(file_path, disabled_tests, curpath, file_ext):
    with open(file_path) as f:
        for i, line in enumerate(f.readlines()):
            line = line.strip()
            # Check for TODO and language-specific flags identifying a disabled tests
            if TODO_ENABLE.match(line) or any(
                pattern in line for pattern in DISABLED_FLAG_MAP.get(file_ext, [])
            ):
                bug = FXB_REGEX.search(line)
                date = str(datetime.date.today())  # E.g. "2021-05-27".
                disabled_tests.append(
                    {
                        "path": os.path.relpath(file_path, curpath),
                        "line_number": i + 1,
                        "bug": bug.group(1) if bug else "",
                        "compiled": file_ext is BUILD_EXT,
                        "date_discovered": date,
                    }
                )


class TestFindDisabledTest(unittest.TestCase):
    def setUp(self):
        self.tempdir = tempfile.TemporaryDirectory("find_disabled_tests")

    def test_cc_file(self):
        got = []
        path = os.path.join(self.tempdir.name, "test.cc")
        date = str(datetime.date.today())  # E.g. "2021-05-27".

        with open(path, "w") as f:
            f.write(
                "\n".join(
                    [
                        "// TODO(fxbug.dev/12345) re-enable this test when de-flaked",
                        "fn flaky_test_we_need_to_fix() { ... }",
                        "TEST_F(BlobfsTest, DISABLED_CorruptBlobNotify) { RunBlobCorruptionTest(); }",
                    ]
                )
            )

        find_disabled_tests(path, got, self.tempdir.name, ".cc")

        want = [
            {
                "path": "test.cc",
                "line_number": 1,
                "bug": "12345",
                "compiled": False,
                "date": date,
            },
            {
                "path": "test.cc",
                "line_number": 3,
                "bug": "",
                "compiled": False,
                "date": date,
            },
        ]
        self.assertEqual(got, want)

    def test_rs_file(self):
        got = []
        path = os.path.join(self.tempdir.name, "test.rs")
        date = str(datetime.date.today())  # E.g. "2021-05-27".

        with open(path, "w") as f:
            f.write(
                "\n".join(
                    [
                        "use std::thread;",
                        "\n",
                        "#[test]",
                        "#[ignore]",
                        "fn works() {",
                        "const N: usize = 4;",
                        "}",
                    ]
                )
            )
        find_disabled_tests(path, got, self.tempdir.name, ".gn")
        want = [
            {
                "path": "test.rs",
                "line_number": 1,
                "bug": "",
                "compiled": False,
                "date": date,
            },
        ]
        self.assertEqual(got, want)

    def tearDown(self):
        self.tempdir.cleanup()


if __name__ == "__main__":
    main()