tests/checkers/unittest_unicode/unittest_bad_chars.py

# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
# For details: https://github.com/pylint-dev/pylint/blob/main/LICENSE
# Copyright (c) https://github.com/pylint-dev/pylint/blob/main/CONTRIBUTORS.txt

# pylint: disable=redefined-outer-name

from __future__ import annotations

import itertools
from collections.abc import Callable
from pathlib import Path
from typing import cast

import astroid
import pytest
from astroid import AstroidBuildingError, nodes

import pylint.checkers.unicode
import pylint.interfaces
import pylint.testutils

from . import CODEC_AND_MSG, FakeNode


@pytest.fixture()
def bad_char_file_generator(tmp_path: Path) -> Callable[[str, bool, str], Path]:
    """Generates a test file for bad chars.

    The generator also ensures that file generated is correct
    """

    def encode_without_bom(string: str, encoding: str) -> bytes:
        return pylint.checkers.unicode._encode_without_bom(string, encoding)

    # All lines contain a not extra checked invalid character
    lines = (
        "# Example File containing bad ASCII",
        "# invalid char backspace: \b",
        "# Bad carriage-return \r # not at the end",
        "# Invalid char sub: \x1A",
        "# Invalid char esc: \x1B",
    )

    def _bad_char_file_generator(
        codec: str, add_invalid_bytes: bool, line_ending: str
    ) -> Path:
        byte_suffix = b""
        if add_invalid_bytes:
            if codec == "utf-8":
                byte_suffix = b"BAD:\x80abc"
            elif codec == "utf-16":
                byte_suffix = b"BAD:\n"  # Generates Truncated Data
            else:
                byte_suffix = b"BAD:\xc3\x28 "
            byte_suffix = encode_without_bom(" foobar ", codec) + byte_suffix

        line_ending_encoded = encode_without_bom(line_ending, codec)

        # Start content with BOM / codec definition and two empty lines
        content = f"# coding: {codec} \n # \n ".encode(codec)

        # Generate context with the given codec and line ending
        for lineno, line in enumerate(lines):
            byte_line = encode_without_bom(line, codec)
            byte_line += byte_suffix + line_ending_encoded
            content += byte_line

            # Directly test the generated content
            if not add_invalid_bytes:
                # Test that the content is correct and gives no errors
                try:
                    byte_line.decode(codec, "strict")
                except UnicodeDecodeError as e:
                    raise ValueError(
                        f"Line {lineno} did raise unexpected error: {byte_line!r}\n{e}"
                    ) from e
            else:
                try:
                    # But if there was a byte_suffix we expect an error
                    # because that is what we want to test for
                    byte_line.decode(codec, "strict")
                except UnicodeDecodeError:
                    ...
                else:
                    raise ValueError(
                        f"Line {lineno} did not raise decode error: {byte_line!r}"
                    )

        file = tmp_path / "bad_chars.py"
        file.write_bytes(content)
        return file

    return _bad_char_file_generator


class TestBadCharsChecker(pylint.testutils.CheckerTestCase):
    CHECKER_CLASS = pylint.checkers.unicode.UnicodeChecker

    checker: pylint.checkers.unicode.UnicodeChecker

    @pytest.mark.parametrize(
        "codec_and_msg, line_ending, add_invalid_bytes",
        [
            pytest.param(
                codec_and_msg,
                line_ending[0],
                suffix[0],
                id=f"{codec_and_msg[0]}_{line_ending[1]}_{suffix[1]}",
            )
            for codec_and_msg, line_ending, suffix in itertools.product(
                CODEC_AND_MSG,
                (("\n", "linux"), ("\r\n", "windows")),
                ((False, "valid_line"), (True, "not_decode_able_line")),
            )
            # Only utf8 can drop invalid lines
            if codec_and_msg[0].startswith("utf") or not suffix[0]
        ],
    )
    def test_find_bad_chars(
        self,
        bad_char_file_generator: Callable[[str, bool, str], Path],
        codec_and_msg: tuple[str, tuple[pylint.testutils.MessageTest]],
        line_ending: str,
        add_invalid_bytes: bool,
    ) -> None:
        """All combinations of bad characters that are accepted by Python at the moment
        are tested in all possible combinations of
          - line ending
          - encoding
          - including not encode-able byte (or not)
        """
        codec, start_msg = codec_and_msg

        start_lines = 2

        file = bad_char_file_generator(codec, add_invalid_bytes, line_ending)

        try:
            # We need to use ast from file as only this function reads bytes and not
            # string
            module = astroid.MANAGER.ast_from_string(file)
        except AstroidBuildingError:
            # pylint: disable-next=redefined-variable-type
            module = cast(nodes.Module, FakeNode(file.read_bytes()))

        expected = [
            *start_msg,
            pylint.testutils.MessageTest(
                msg_id="invalid-character-backspace",
                line=2 + start_lines,
                end_line=2 + start_lines,
                # node=module,
                args=None,
                confidence=pylint.interfaces.HIGH,
                col_offset=27,
                end_col_offset=28,
            ),
            pylint.testutils.MessageTest(
                msg_id="invalid-character-carriage-return",
                line=3 + start_lines,
                end_line=3 + start_lines,
                # node=module,
                args=None,
                confidence=pylint.interfaces.HIGH,
                col_offset=23,
                end_col_offset=24,
            ),
            pylint.testutils.MessageTest(
                msg_id="invalid-character-sub",
                line=4 + start_lines,
                end_line=4 + start_lines,
                # node=module,
                args=None,
                confidence=pylint.interfaces.HIGH,
                col_offset=21,
                end_col_offset=22,
            ),
            pylint.testutils.MessageTest(
                msg_id="invalid-character-esc",
                line=5 + start_lines,
                end_line=5 + start_lines,
                # node=module,
                args=None,
                confidence=pylint.interfaces.HIGH,
                col_offset=21,
                end_col_offset=22,
            ),
        ]
        with self.assertAddsMessages(*expected):
            self.checker.process_module(module)

    @pytest.mark.parametrize(
        "codec_and_msg, char, msg_id",
        [
            pytest.param(
                codec_and_msg,
                char_msg[0],
                char_msg[1],
                id=f"{char_msg[1]}_{codec_and_msg[0]}",
            )
            for codec_and_msg, char_msg in itertools.product(
                CODEC_AND_MSG,
                (
                    ("\0", "invalid-character-nul"),
                    ("\N{ZERO WIDTH SPACE}", "invalid-character-zero-width-space"),
                ),
            )
            # Only utf contains zero width space
            if (
                char_msg[0] != "\N{ZERO WIDTH SPACE}"
                or codec_and_msg[0].startswith("utf")
            )
        ],
    )
    def test_bad_chars_that_would_currently_crash_python(
        self,
        char: str,
        msg_id: str,
        codec_and_msg: tuple[str, tuple[pylint.testutils.MessageTest]],
    ) -> None:
        """Special test for a file containing chars that lead to
        Python or Astroid crashes (which causes Pylint to exit early).
        """
        codec, start_msg = codec_and_msg
        # Create file that will fail loading in astroid.
        # We still want to check this, in case this behavior changes
        content = f"# # coding: {codec}\n# file containing {char} <-\n"
        module = FakeNode(content.encode(codec))

        expected = [
            *start_msg,
            pylint.testutils.MessageTest(
                msg_id=msg_id,
                line=2,
                end_line=2,
                # node=module,
                args=None,
                confidence=pylint.interfaces.HIGH,
                col_offset=19,
                end_col_offset=20,
            ),
        ]

        with self.assertAddsMessages(*expected):
            self.checker.process_module(cast(nodes.Module, module))

    @pytest.mark.parametrize(
        "char, msg, codec",
        [
            pytest.param(
                char.unescaped,
                char.human_code(),
                codec_and_msg[0],
                id=f"{char.name}_{codec_and_msg[0]}",
            )
            for char, codec_and_msg in itertools.product(
                pylint.checkers.unicode.BAD_CHARS, CODEC_AND_MSG
            )
            # Only utf contains zero width space
            if (
                char.unescaped != "\N{ZERO WIDTH SPACE}"
                or codec_and_msg[0].startswith("utf")
            )
        ],
    )
    def test___check_invalid_chars(self, char: str, msg: str, codec: str) -> None:
        """Check function should deliver correct column no matter which codec we used."""
        with self.assertAddsMessages(
            pylint.testutils.MessageTest(
                msg_id=msg,
                line=55,
                args=None,
                confidence=pylint.interfaces.HIGH,
                end_line=55,
                col_offset=5,
                end_col_offset=6,
            )
        ):
            self.checker._check_invalid_chars(f"#234{char}".encode(codec), 55, codec)