Utilities/Scripts/regenerate-presets.py

#!/usr/bin/env python3
"""Generate CMake presets documentation (reST) and JSON schema.

The documentation and JSON schema for CMake presets are both generated from
Help/manual/presets/schema.yaml, which describes the schema in a way that
allows both documentation forms to be combined in a single file, and also makes
it much easier to manage version revisions (compared to editing the JSON schema
'by hand').

All input and output files are expected to be in known locations relative to
this script.

Usage: python3 regenerate-presets.py
"""

import json
import re
from copy import deepcopy
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Self

import yaml

LATEST = 1
TYPES = {}
REFS = {}

PRESETS_REL = 'Help/manual/presets'
SCHEMA_YAML_FILENAME = 'schema.yaml'
SCHEMA_JSON_FILENAME = 'schema.json'

WORKSPACE = Path(__file__).parent.parent.parent.absolute()
SCRIPT_REL = Path(__file__).relative_to(WORKSPACE)
PRESETS = WORKSPACE / PRESETS_REL
DIAGNOSTICS = WORKSPACE / 'Source/cmDiagnostics.h'

RST_BANNER = f"""\
.. This file was generated by {SCRIPT_REL.as_posix()}
   from {PRESETS_REL}/{SCHEMA_YAML_FILENAME}. Do not edit.
"""

DIAGNOSTIC_TABLE_MACRO = 'CM_FOR_EACH_DIAGNOSTIC_TABLE'
CONFIGURE_PRESET_PROPERTIES_PATH = (
    'properties', 'configurePresets', 'items', 'properties'
)
WARNING_DESCRIPTION = (
    'An optional boolean. '
    'Equivalent to passing -W{c} or -Wno-{c} on the command line. '
    'This may not be set to false if errors.{p} is set to true.'
)
WARNING_SPHINX_DESCRIPTION = """
An optional boolean. Equivalent to passing :option:`-W{c} <cmake -W>` or
:option:`-Wno-{c} <cmake -Wno->` on the command line.
This may not be set to ``false`` if ``errors.{p}`` is set to ``true``.
""".strip()
ERROR_DESCRIPTION = (
    'An optional boolean. '
    'Equivalent to passing -Werror={c} or -Wno-error={c} on the command line. '
    'This may not be set to true if warnings.{p} is set to false.'
)
ERROR_SPHINX_DESCRIPTION = """
An optional boolean. Equivalent to passing :cmake-option:`-Werror={c}` or
:cmake-option:`-Wno-error={c}` on the command line.
This may not be set to ``true`` if ``warnings.{p}`` is set to ``false``.
""".strip()


# =============================================================================
@dataclass
class Diagnostic:
    presetName: str
    cliName: str
    since: int

    # -------------------------------------------------------------------------
    def format(self, template: str) -> str:
        return template.format(c=self.cliName, p=self.presetName)


# =============================================================================
class Value:

    # -------------------------------------------------------------------------
    def __init__(self, data: list[dict[str, Any]]):
        self.data = data
        self.spans = None

    # -------------------------------------------------------------------------
    @property
    def objects(self) -> dict[str, 'Object']:
        return {}

    # -------------------------------------------------------------------------
    def updateSpans(self, since: int, until: int) -> None:
        self.spans = [(since, until)]

    # -------------------------------------------------------------------------
    def updateRefs(self, objects: dict[str, 'Object']) -> None:
        pass

    # -------------------------------------------------------------------------
    def defs(self) -> dict[str, Any]:
        return {}

    # -------------------------------------------------------------------------
    def schema(self, version: int) -> dict[str, Any]:
        return deepcopy(self.data)


# =============================================================================
class Object(Value):

    # -------------------------------------------------------------------------
    def __init__(self, data: dict[str, Any], name: str,
                 label: str | None = None):
        # Prevent data sharing from modifying unrelated parts of the schema as
        # we pick apart our section.
        data = deepcopy(data)

        self.name = data.pop('id', name)
        self.label = label

        self.properties = {}
        for n, p in data.pop('properties').items():
            pl = '.'.join([label, n]) if label is not None else n
            self.properties[n] = Property(p, n, pl)
        if len(self.properties):
            self.properties[CommentProperty.name] = CommentProperty()

        ap = data.pop('additionalProperties', None)
        if ap is not None:
            self.additionalProperties = buildValue(ap, self.name, self.label)
        else:
            self.additionalProperties = None

        super().__init__(data)

    # -------------------------------------------------------------------------
    @property
    def objects(self) -> dict[str, Self]:
        out = {self.name: self}
        for p in self.properties.values():
            out.update(p.objects)
        if self.additionalProperties is not None:
            out.update(self.additionalProperties.objects)
        return out

    # -------------------------------------------------------------------------
    def updateSpans(self, since: int, until: int) -> None:
        spans = [[(since, until)]]
        for p in self.properties.values():
            p.updateSpans(since, until)
            spans.append(p.spans)
        if self.additionalProperties is not None:
            self.additionalProperties.updateSpans(since, until)
            spans.append(self.additionalProperties.spans)
        self.spans = combineSpans(*spans)

    # -------------------------------------------------------------------------
    def updateRefs(self, objects: dict[str, 'Object']) -> None:
        for p in self.properties.values():
            p.updateRefs(objects)
        if self.additionalProperties is not None:
            self.additionalProperties.updateRefs(objects)

    # -------------------------------------------------------------------------
    def doc(self, since: int, until: int) -> str | None:
        since = max(since, self.spans[0][0])
        until = min(until, self.spans[-1][1])

        out = []
        for _, p in self.properties.items():
            content = p.doc(since, until)
            if not content:
                continue
            out.append(content)

        return '\n\n'.join(out) if len(out) else None

    # -------------------------------------------------------------------------
    def defs(self, exclude: set[str] = {}) -> dict[str, Any]:
        out = {}
        for n, p in self.properties.items():
            if n in exclude:
                continue
            for s in p.spans:
                r = f'{p.label}@{spanRef(*s)}'
                out[r] = p.schema(s[0])
            out.update(p.defs())
        if self.additionalProperties is not None:
            out.update(self.additionalProperties.defs())
        return out

    # -------------------------------------------------------------------------
    def schema(self, version: int, inline: set[str] = {}) -> dict[str, Any]:
        out = deepcopy(self.data)

        props = {}
        required = []
        for n, p in self.properties.items():
            if n in inline:
                props[n] = p.schema(version)
            else:
                r = p.ref(version)
                if r is not None:
                    props[n] = r
            if p.required:
                required.append(n)

        out['properties'] = props
        if len(required):
            out['required'] = required

        if self.additionalProperties is not None:
            ap = self.additionalProperties.schema(version)
            out['additionalProperties'] = ap
        return out


# =============================================================================
class Ref(Value):

    # -------------------------------------------------------------------------
    def __init__(self, data: list[dict[str, Any]]):
        self.target = data.pop('target')
        self.targetSpans = None
        super().__init__(data)

    # -------------------------------------------------------------------------
    def updateSpans(self, since: int, until: int) -> None:
        if self.spans is None:
            super().updateSpans(since, until)
        self.spans = trimSpans(self.spans, since, until)

    # -------------------------------------------------------------------------
    def updateRefs(self, objects: dict[str, 'Object']) -> None:
        global REFS

        t = objects.get(self.target)
        if t is not None:
            since = self.spans[0][0]
            until = self.spans[-1][1]
            self.spans = trimSpans(t.spans, since, until)
            self.targetSpans = t.spans

        rs = REFS.get(self.target, (since, until))
        REFS[self.target] = (min(since, rs[0]), max(until, rs[1]))

    # -------------------------------------------------------------------------
    def schema(self, version: int) -> dict[str, Any]:
        ts = trimSpans(self.targetSpans, *REFS[self.target])
        s = findSpan(ts, version)
        if s is not None:
            return {
                '$ref': f'#/definitions/{self.target}@{spanRef(*s)}',
            }
        return {}


# =============================================================================
class Array(Value):

    # -------------------------------------------------------------------------
    def __init__(self, data: dict[str, Any], name: str, label: str):
        item = data.pop('items')
        self.value = buildValue(item, name, label)

        super().__init__(data)

    # -------------------------------------------------------------------------
    @property
    def objects(self) -> dict[str, Object]:
        return self.value.objects

    # -------------------------------------------------------------------------
    def updateSpans(self, since: int, until: int) -> None:
        self.value.updateSpans(since, until)
        self.spans = self.value.spans

    # -------------------------------------------------------------------------
    def updateRefs(self, objects: dict[str, 'Object']) -> None:
        self.value.updateRefs(objects)

    # -------------------------------------------------------------------------
    def defs(self) -> dict[str, Any]:
        return self.value.defs()

    # -------------------------------------------------------------------------
    def schema(self, version: int) -> dict[str, Any]:
        out = deepcopy(self.data)
        out['items'] = self.value.schema(version)
        return out


# =============================================================================
class Union(Value):

    # -------------------------------------------------------------------------
    def __init__(self, data: list[dict[str, Any]], name: str, label: str):
        self.items = []
        for i, d in enumerate(data):
            self.items.append(buildValue(d, name, self.itemLabel(label, i)))

        super().__init__({})

    # -------------------------------------------------------------------------
    def itemLabel(self, label: str, index: int) -> str:
        return label

    # -------------------------------------------------------------------------
    @property
    def objects(self) -> dict[str, Object]:
        out = {}
        for v in self.items:
            out.update(v.objects)
        return out

    # -------------------------------------------------------------------------
    def updateSpans(self, since: int, until: int) -> None:
        spans = [[(since, until)]]
        for i in self.items:
            i.updateSpans(since, until)
            spans.append(i.spans)
        self.spans = combineSpans(*spans)

    # -------------------------------------------------------------------------
    def updateRefs(self, objects: dict[str, 'Object']) -> None:
        for i in self.items:
            i.updateRefs(objects)

    # -------------------------------------------------------------------------
    def defs(self) -> dict[str, Any]:
        out = {}
        for v in self.items:
            out.update(v.defs())
        return out

    # -------------------------------------------------------------------------
    def schema(self, version: int) -> dict[str, Any]:
        out = []
        for i in self.items:
            out.append(i.schema(version))
        return {'anyOf': out}


# =============================================================================
class Variant(Union):

    # -------------------------------------------------------------------------
    def __init__(self, data: list[dict[str, Any]], name: str, label: str):
        self.identifier = data.pop('id', None)

        if self.identifier is not None:
            name = self.identifier
            label = self.identifier

        super().__init__(data.pop('items'), name, label)

    # -------------------------------------------------------------------------
    def itemLabel(self, label: str, index: int) -> str:
        return f'{label}[{index}]'

    # -------------------------------------------------------------------------
    @property
    def objects(self) -> dict[str, Object]:
        out = super().objects
        if self.identifier is not None:
            out[self.identifier] = self
        return out

    # -------------------------------------------------------------------------
    def doc(self, since: int, until: int) -> None:
        return None

    # -------------------------------------------------------------------------
    def defs(self) -> dict[str, Any]:
        out = super().defs()

        if self.identifier is not None:
            for s in self.spans:
                n = f'{self.identifier}@{spanRef(*s)}'
                out[n] = super().schema(s[0])

        return out

    # -------------------------------------------------------------------------
    def schema(self, version: int) -> dict[str, Any]:
        if self.identifier is not None:
            s = findSpan(self.spans, version)
            if s is not None:
                return {
                    '$ref': f'#/definitions/{self.identifier}@{spanRef(*s)}',
                }
            return {}

        return super().schema(version)


# =============================================================================
class Property:

    # -------------------------------------------------------------------------
    def __init__(self, data: dict[str, Any], name: str, label: str):
        self.name = name
        self.label = label
        self.since = data.pop('since', 1)
        self.until = data.pop('until', LATEST + 1)
        self.required = data.pop('required', False)
        self.content = data.pop('sphinxDescription', data.get('description'))

        note = data.pop('sphinxNote', None)
        if note is not None:
            self.content += f'\n\n.. note::\n\n{indent(note)}'

        self._revisions = []
        for rev in data.pop('revisions', []):
            rd = deepcopy(data)
            rd.update(rev)
            self._revisions.append(Property(rd, self.name, self.label))

        if len(self._revisions):
            self.since = max(self.since,
                             min([r.since for r in self._revisions]))
            self.until = min(self.until,
                             max([r.until for r in self._revisions]))
            self.value = None
        else:
            self.value = buildValue(data, self.name, self.label)

        self.spans = None

    # -------------------------------------------------------------------------
    @property
    def objects(self) -> dict[str, Object]:
        if len(self._revisions):
            out = {}
            for r in self._revisions:
                out.update(r.objects)
            return out

        return self.value.objects

    # -------------------------------------------------------------------------
    def updateSpans(self, since: int, until: int) -> None:
        since = max(since, self.since)
        until = min(until, self.until)

        if len(self._revisions):
            spans = [[(since, until)]]
            for r in self._revisions:
                r.updateSpans(since, until)
                spans.append(r.spans)
            self.spans = combineSpans(*spans)
        else:
            self.value.updateSpans(since, until)
            self.spans = self.value.spans

    # -------------------------------------------------------------------------
    def updateRefs(self, objects: dict[str, 'Object']) -> None:
        if len(self._revisions):
            for r in self._revisions:
                r.updateRefs(objects)
        else:
            self.value.updateRefs(objects)

    # -------------------------------------------------------------------------
    def ref(self, version: int) -> dict[str, str] | None:
        s = findSpan(self.spans, version)
        if s is not None:
            return {
                '$ref': f'#/definitions/{self.label}@{spanRef(*s)}',
            }
        return None

    # -------------------------------------------------------------------------
    def doc(self, since: int, until: int) -> str | None:
        if self.content is None:
            return None

        out = f'.. _`CMakePresets.{self.label}`:\n\n``{self.name}``\n'
        if self.since > since:
            out += f'  .. presets-versionadded:: {self.since}\n\n'
        if self.until < until:
            out += f'  .. presets-versionremoved:: {self.until}\n\n'
        out += indent(self.content)

        return out

    # -------------------------------------------------------------------------
    def defs(self) -> dict[str, Any]:
        if len(self._revisions):
            out = {}
            for r in self._revisions:
                out.update(r.defs())
            return out

        return self.value.defs()

    # -------------------------------------------------------------------------
    def schema(self, version: int) -> dict[str, Any]:
        for r in self._revisions:
            if version >= r.since and version < r.until:
                return r.value.schema(version)
        return self.value.schema(version)


# =============================================================================
class CommentProperty(Property):
    name: str = '$comment'
    since: int = 10

    # -------------------------------------------------------------------------
    def __init__(self):
        data = {
            'since': type(self).since,
            'id': 'comment',
            'type': 'variant',
            'items': [
                {
                    'type': 'string',
                    'description': 'A single-line comment.',
                },
                {
                    'type': 'array',
                    'description': 'A multi-line comment.',
                    'minItems': 1,
                    'items': {
                        'type': 'string',
                        'description': 'One line of the multi-line comment.',
                    }
                }
            ],
        }
        super().__init__(data, type(self).name, type(self).name)

    # -------------------------------------------------------------------------
    def doc(self, since: int, until: int) -> None:
        return None


# -----------------------------------------------------------------------------
def buildValue(
    data: Any, name: str, label: str | None = None
) -> Variant | Object | Array | Ref | Value:
    if type(data) is dict:
        if 'anyOf' in data:
            return Union(data.pop('anyOf'), name, label)
        elif data['type'] == 'variant':
            return Variant(data, name, label)
        elif data['type'] == 'array':
            return Array(data, name, label)
        elif data['type'] == 'object':
            return Object(data, name, label)
        elif data['type'] == 'ref':
            return Ref(data)

    return Value(data)


# -----------------------------------------------------------------------------
def extractDivisions(spans: list[tuple[int, int]]) -> set[int]:
    return {n for s in spans for n in s}


# -----------------------------------------------------------------------------
def trimSpans(spans: list[tuple[int, int]],
              since: int, until: int) -> list[tuple[int, int]]:
    divisions = extractDivisions(spans)

    last = since
    out = []

    for d in sorted(divisions):
        if d > since:
            out.append((last, min(d, until)))
            last = min(d, until)
        if d >= until:
            break

    return out


# -----------------------------------------------------------------------------
def combineSpans(*spans: list[tuple[int, int]]) -> list[tuple[int, int]]:
    divisions = set()
    for s in spans:
        divisions.update(extractDivisions(s))

    divisions = sorted(divisions)
    last = divisions.pop(0)
    out = []

    for d in divisions:
        out.append((last, d))
        last = d

    return out


# -----------------------------------------------------------------------------
def findSpan(spans: list[tuple[int, int]],
             version: int) -> tuple[int, int] | None:
    for s in spans:
        if version >= s[0] and version < s[1]:
            return s

    return None


# -----------------------------------------------------------------------------
def spanRef(since: int, until: int) -> str:
    if since + 1 < until:
        if until > LATEST:
            return f'v{since}..'
        else:
            return f'v{since}..v{until - 1}'
    return f'v{since}'


# -----------------------------------------------------------------------------
def indent(text: str):
    return '\n'.join([('  ' + line).rstrip() for line in text.split('\n')])


# -----------------------------------------------------------------------------
def trimDescriptions(data: Any):
    if type(data) is dict:
        for k, v in data.items():
            if type(v) is str and k in {'description', 'sphinxDescription'}:
                data[k] = v.strip()
            elif type(v) is dict:
                data[k] = trimDescriptions(v)
            elif type(v) is list:
                data[k] = [trimDescriptions(item) for item in v]

    return data


# -----------------------------------------------------------------------------
def getPath(data: Any, path: tuple[str]) -> Any:
    if len(path) == 0:
        return data
    return getPath(data[path[0]], path[1:])


# -----------------------------------------------------------------------------
def diagnosticPresetName(symbol: str) -> str:
    sep = False
    out = ''
    for c in symbol[4:]:
        if c == '_':
            sep = True
        elif sep:
            out += c
            sep = False
        else:
            out += c.lower()
    return out


# -----------------------------------------------------------------------------
def buildDiagnosticsSchema(
    diagnostics: list[Diagnostic],
    descriptionTemplate: str,
    sphinxDescriptionTemplate: str,
) -> dict[str, dict[str, Any]]:
    out = {}

    for d in diagnostics:
        out[d.presetName] = {
            'since': d.since,
            'type': 'boolean',
            'description': d.format(descriptionTemplate),
            'sphinxDescription': d.format(sphinxDescriptionTemplate),
        }

    return out


# -----------------------------------------------------------------------------
def mergeDiagnostics(
    generated: dict[str, dict[str, Any]],
    extra: dict[str, dict[str, Any]],
) -> dict[str, dict[str, Any]]:
    # Get unsorted names.
    tail = {}
    for k in list(extra.keys()):
        s = extra[k].pop('sort', True)
        if not s:
            tail[k] = extra.pop(k)

    # Combine inputs and sort.
    out = {}
    combined = deepcopy(generated)
    combined.update(extra)
    for k in sorted(combined.keys()):
        out[k] = combined[k]

    # Add unsorted items and return result.
    out.update(tail)
    return out


# -----------------------------------------------------------------------------
def readDiagnostics(path: Path | str) -> list[Diagnostic]:
    content = ''

    # Extract diagnostics table from header.
    with open(path, 'r', encoding='utf-8') as f:
        extracting = False
        for line in f:
            if DIAGNOSTIC_TABLE_MACRO in line:
                extracting = True
                continue

            if extracting:
                content += line.replace('\\\n', ' ')
                if not line.strip().endswith('\\'):
                    break

    out = []
    while True:
        m = re.match(r'\s*SELECT[(]([^)]+)[)]', content)
        if m is None:
            break

        args = [a.strip() for a in m.group(1).split(',')]
        p = diagnosticPresetName(args[3])
        c = args[3][4:].lower().replace('_', '-')
        v = int(args[4])
        if v > 1:
            out.append(Diagnostic(p, c, v))

        content = content[m.span()[1]:]

    return out


# -----------------------------------------------------------------------------
def main():
    # Read the schema definition.
    with open(PRESETS / SCHEMA_YAML_FILENAME, 'r', encoding='utf-8') as f:
        schema = yaml.safe_load(f)

    # Extract the current (latest) version.
    global LATEST

    del schema['definitions']
    LATEST = schema.pop('version')

    # Remove whitespace around descriptions.
    schema = trimDescriptions(schema)

    # Read diagnostics and update schema.
    diagnostics = readDiagnostics(DIAGNOSTICS)
    configureSchema = getPath(schema, CONFIGURE_PRESET_PROPERTIES_PATH)
    configureSchema['warnings']['properties'] = mergeDiagnostics(
        buildDiagnosticsSchema(diagnostics, WARNING_DESCRIPTION,
                               WARNING_SPHINX_DESCRIPTION),
        configureSchema['warnings']['properties'])
    configureSchema['errors']['properties'] = mergeDiagnostics(
        buildDiagnosticsSchema(diagnostics, ERROR_DESCRIPTION,
                               ERROR_SPHINX_DESCRIPTION),
        configureSchema['errors']['properties'])

    # Extract global type definitions.
    global TYPES, REFS

    for t in schema.pop('types', []):
        name = t['id']
        value = buildValue(t, name)
        value.updateSpans(1, LATEST + 1)
        TYPES[name] = value

    for t in TYPES.values():
        t.updateRefs(TYPES)

    # Reset the spans for references, as these currently contain only spans for
    # circular references of global types, which are likely broader than the
    # actual usage spans of those types. We'll recalculate the correct spans
    # when we update refs from the root object.
    REFS = {}

    # Parse the root object. This will also recursively parse other objects.
    root = Object(schema, 'root')
    root.updateSpans(1, LATEST + 1)

    # Resolve references.
    objects = root.objects

    types = objects
    types.update(TYPES)

    root.updateRefs(types)

    for name, value in TYPES.items():
        span = REFS.get(name)
        if span is not None:
            value.updateSpans(*span)

    root.updateSpans(1, LATEST + 1)

    # Extract Sphinx documentation for each object's properties.
    for n, o in objects.items():
        doc = o.doc(1, LATEST + 1)
        if doc is not None:
            print(f'- Generating reST documentation for {n} properties')
            with open(PRESETS / f'{o.name}-properties.rst', 'w', encoding='utf-8') as f:
                print(RST_BANNER, file=f)
                print(doc, file=f)

    # Generate schema for each version.
    versions = []
    for v in range(1, LATEST + 1):
        vs = root.schema(v, inline={'version'})
        vs['properties']['version']['const'] = v
        versions.append(vs)

    # Add definitions and write the schema.
    types = root.defs(exclude={'version'})
    for t in TYPES.values():
        types.update(t.defs())

    schema = {
        '$schema': 'http://json-schema.org/draft/2020-12/schema#',
        'type': 'object',
        'oneOf': versions,
        'required': ['version'],
        'definitions': types
    }

    with open(PRESETS / SCHEMA_JSON_FILENAME, 'w', encoding='utf-8') as f:
        print(json.dumps(schema, indent=2, ensure_ascii=False), file=f)


# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

main()