run-tests.py

#!/usr/bin/env python

from argparse import ArgumentParser
from logging import basicConfig, INFO, getLogger
from os import name
from pathlib import Path
from subprocess import Popen
from sys import executable

from colorama import init, Fore
from pyjson5 import decode_io


argparser = ArgumentParser(description='Run JSON5 parser tests')
argparser.add_argument('tests', nargs='?', type=Path, default=Path('third-party/json5-tests'))

suffix_implies_success = {
    '.json': True,
    '.json5': True,
    '.txt': False,
}

if __name__ == '__main__':
    basicConfig(level=INFO)
    logger = getLogger(__name__)

    init()

    if name != 'nt':
        code_severe = Fore.RED + '😱'
        code_good = Fore.CYAN + '😄'
        code_bad = Fore.YELLOW + '😠'
    else:
        code_severe = Fore.RED + 'SEVERE'
        code_good = Fore.CYAN + 'GOOD'
        code_bad = Fore.YELLOW + 'BAD'

    good = 0
    bad = 0
    severe = 0

    args = argparser.parse_args()
    index = 0
    for path in sorted(args.tests.glob('*/*.*')):
        kind = path.suffix.split('.')[-1]
        expect_success = suffix_implies_success.get(path.suffix)
        if expect_success is None:
            continue

        index += 1
        category = path.parent.name
        name = path.stem
        try:
            p = Popen((executable, 'transcode-to-json.py', str(path)))
            outcome = p.wait(5)
        except Exception:
            logger.error('Error while testing: %s', path, exc_info=True)
            severe += 1
            continue

        is_success = outcome == 0
        is_failure = outcome == 1
        is_severe = outcome not in (0, 1)
        is_good = is_success if expect_success else is_failure
        code = code_severe if is_severe else code_good if is_good else code_bad
        print(
            '#', index, ' ', code, ' '
            'Category <', category, '> | '
            'Test <', name, '> | '
            'Data <', kind, '> | '
            'Expected <', 'pass' if expect_success else 'FAIL', '> | '
            'Actual <', 'pass' if is_success else 'FAIL', '>',
            Fore.RESET,
            sep='',
        )
        if is_severe:
            severe += 1
        elif is_good:
            good += 1
        else:
            bad += 1

    is_severe = severe > 0
    is_good = bad == 0
    code = code_severe if is_severe else code_good if is_good else code_bad
    print()
    print(
        code, ' ',
        good, ' × correct outcome | ',
        bad, ' × wrong outcome | ',
        severe, ' × severe errors',
        Fore.RESET,
        sep=''
    )
    raise SystemExit(2 if is_severe else 0 if is_good else 1)