mlir/utils/pygments/mlir_lexer.py

# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

from pygments.lexer import RegexLexer, bygroups, include, using
from pygments.token import *
import re


class MlirLexer(RegexLexer):
    """Pygments lexer for MLIR.

    This lexer focuses on accurate tokenization of common MLIR constructs:
    - SSA values (%%... / %...)
    - attribute and type aliases (#name =, !name =)
    - types (builtin and dialect types, parametric types)
    - attribute dictionaries and nested containers to a reasonable depth
    - numbers (ints, floats with exponents, hex)
    - strings with common escapes
    - line comments (// ...)
    - block labels (^foo) and operations
    """

    name = "MLIR"
    aliases = ["mlir"]
    filenames = ["*.mlir"]

    flags = re.MULTILINE

    class VariableList(RegexLexer):
        """Lexer for lists of SSA variables separated by commas."""

        tokens = {
            "root": [
                (r"\s+", Text),
                (r",", Punctuation),
                (r"%[_A-Za-z0-9\.\$\-:#]+", Name.Variable),
            ]
        }

    tokens = {
        "root": [
            # Comments
            (r"//.*?$", Comment.Single),
            # operation name with assignment: %... = op.name
            (
                r"^(\s*)(%[\%_A-Za-z0-9\:#\,\s]+)(=)(\s*)([A-Za-z0-9_\.\$\-]+)\b",
                bygroups(Text, using(VariableList), Operator, Text, Name.Builtin),
            ),
            # operation name without result
            (r"^(\s*)([A-Za-z0-9_\.\$\-]+)\b(?=[^<:])", bygroups(Text, Name.Builtin)),
            # Attribute alias definition:  #name =
            (
                r"^(\s*)(#[_A-Za-z0-9\$\-\.]+)(\b)(\s*=)",
                bygroups(Text, Name.Constant, Text, Operator),
            ),
            # Type alias definition: !name =
            (
                r"^(\s*)(![_A-Za-z0-9\$\-\.]+)(\b)(\s*=)",
                bygroups(Text, Keyword.Type, Text, Operator),
            ),
            # SSA values (uses)
            (r"%[_A-Za-z0-9\.\$\-:#]+", Name.Variable),
            # attribute refs, constants and named attributes
            (r"#[_A-Za-z0-9\$\-\.]+\b", Name.Constant),
            # symbol refs / function-like names
            (r"@[_A-Za-z][_A-Za-z0-9\$\-\.]*\b", Name.Function),
            # blocks
            (r"\^[A-Za-z0-9_\$\.\-]+", Name.Label),
            # types by exclamation or builtin names
            (r"![_A-Za-z0-9\$\-\.]+\b", Keyword.Type),
            # NOTE: please sync changes to corresponding builtin type rule in "angled-type"
            (r"\b(bf16|f16|f32|f64|f80|f128|index|none|(u|s)?i[0-9]+)\b", Keyword.Type),
            # container-like dialect types (tensor<...>, memref<...>, vector<...>)
            (
                r"\b(complex|memref|tensor|tuple|vector)\s*(<)",
                bygroups(Keyword.Type, Punctuation),
                "angled-type",
            ),
            # affine constructs
            (r"\b(affine_map|affine_set)\b", Keyword.Reserved),
            # common builtin operators / functions inside affine_map
            (r"\b(ceildiv|floordiv|mod|symbol)\b", Name.Other),
            # identifiers / bare words
            (r"\b[_A-Za-z][_A-Za-z0-9\.-]*\b", Name.Other),
            # numbers: hex, float (with exponent), integer
            (r"\b0x[0-9A-Fa-f]+\b", Number.Hex),
            (r"\b([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?\b", Number.Float),
            (r"\b[0-9]+\b", Number.Integer),
            # strings
            (r'"', String.Double, "string"),
            # punctuation and arrow-like tokens
            (r"->|>=|<=|\>=|\<=|\->|\=>", Operator),
            (r"[()\[\]{}<>,.:=]", Punctuation),
            # operators
            (r"[-+*/%]", Operator),
        ],
        # string state with common escapes
        "string": [
            (r'\\[ntr"\\]', String.Escape),
            (r'[^"\\]+', String.Double),
            (r'"', String.Double, "#pop"),
        ],
        # angled-type content
        "angled-type": [
            # match nested '<' and '>'
            (r"<", Punctuation, "#push"),
            (r">", Punctuation, "#pop"),
            # dimensions like 3x or 3x3x... and standalone numbers:
            # - match numbers that are followed by an 'x' (dimension separator)
            (r"([0-9]+)(?=(?:x))", Number.Integer),
            # - match bare numbers (sizes)
            (r"[0-9]+", Number.Integer),
            # dynamic dimension '?'
            (r"\?", Name.Integer),
            # the 'x' dimension separator (treat as punctuation)
            (r"x", Punctuation),
            # element / builtin types inside angle brackets (no word-boundary)
            # NOTE: please sync changes to corresponding builtin type rule in "root"
            (
                r"(?:bf16|f16|f32|f64|f80|f128|index|none|(?:[us]?i[0-9]+))",
                Keyword.Type,
            ),
            # also allow nested container-like types to be recognized
            (
                r"\b(complex|memref|tensor|tuple|vector)\s*(<)",
                bygroups(Keyword.Type, Punctuation),
                "angled-type",
            ),
            # fall back to root rules for anything else
            include("root"),
        ],
    }