vscode-language-fidl/tools/generate-syntax.ts

// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// This generates a language definition file because writing it by hand is too hard.

import { OnigRegExp } from "oniguruma";

// Format of language definition JSON
const tmSchema =
  "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json";
type TmCaptures = { [k: string]: { name: string } };
type TmIncludePattern = { include: string };
type TmMatchPattern = {
  name?: string;
  match: string;
  captures?: TmCaptures;
};
type TmBlockPattern = {
  name?: string;
  begin: string;
  beginCaptures?: TmCaptures;
  end: string;
  endCaptures?: TmCaptures;
  patterns: TmPattern[];
};
type TmPattern = TmIncludePattern | TmMatchPattern | TmBlockPattern;
type TmLanguage = {
  $schema: string;
  name: string;
  scopeName: string;
  patterns: TmPattern[];
  repository: {
    [key: string]: { patterns: TmPattern[] };
  };
};

class NamedPattern {
  readonly re: string;
  readonly names: string[];
  constructor(re: string, names?: string[]) {
    this.re = re;
    this.names = names || [];
    // Hack to only validate if there are no unresolved recursive groups
    const recursive_re = /\\g<(.*?)>/g;
    let match;
    while ((match = recursive_re.exec(re)) !== null) {
      if (!re.includes(`(?<${match[1]}>`)) {
        // Group definition not found, can't validate yet.
        return;
      }
    }
    this.validate();
  }

  validate() {
    const num_groups = new OnigRegExp(this.re + "|").searchSync("")!.length - 1;
    if (num_groups !== this.names.length) {
      throw new Error(
        `Found ${num_groups} but expected ${this.names.length} groups in ${
          this.re
        }`
      );
    }
  }

  toString() {
    return this.re;
  }

  assert(s: string) {
    const re = new OnigRegExp(this.re);
    const m = re.searchSync(s);
    if (!m) {
      throw Error(
        `${JSON.stringify(s)} did not match pattern ${JSON.stringify(this.re)}`
      );
    }
    const c = m[0];
    if (c.index !== 0 || c.start !== 0 || c.length !== s.length) {
      throw Error(
        `${JSON.stringify(s)} did not fully match pattern ${JSON.stringify(
          this.re
        )}, only matched ${JSON.stringify(m[0])}`
      );
    }
  }
}

function _captures(pattern: Pattern): TmCaptures {
  const captures: { [k: string]: { name: string } } = {};
  const names = _names(pattern);
  for (let i = 0; i < names.length; i++) {
    captures[`${i + 1}`] = { name: names[i] };
  }
  return captures;
}

function include(name: string): TmIncludePattern {
  return { include: `#${name}` };
}

function match(name: string, pat: Pattern): TmMatchPattern {
  return {
    name: `${name}.fidl`,
    match: pat.toString(),
    captures: _captures(pat)
  };
}

function anonMatch(pattern: Pattern | TmPattern): TmPattern {
  if (pattern instanceof String) {
    return {
      match: pattern.toString()
    };
  }
  if (pattern instanceof NamedPattern) {
    return {
      match: pattern.toString(),
      captures: _captures(pattern)
    };
  }
  return pattern;
}

function block(args: {
  name: string;
  begin: Pattern;
  end: Pattern;
  patterns: Array<Pattern | TmPattern>;
}): TmBlockPattern {
  const tmPatterns: TmPattern[] = args.patterns.map(anonMatch);
  return {
    name: `${args.name}.fidl`,
    begin: args.begin.toString(),
    beginCaptures: _captures(args.begin),
    end: args.end.toString(),
    endCaptures: _captures(args.end),
    patterns: tmPatterns
  };
}

type Pattern = NamedPattern | String;

/** Return names of the groups in the pattern. */
function _names(pat: Pattern | Pattern[]): string[] {
  if (pat instanceof Array) {
    return pat.map(_names).reduce((prev, names) => [...(prev || []), ...names]);
  }
  if (pat instanceof NamedPattern) {
    return pat.names;
  } else {
    return [];
  }
}

function _join(
  pats: Pattern[],
  prefix: string,
  separator: string,
  suffix: string
): NamedPattern {
  const re = pats.map(p => p.toString()).join(separator);
  return new NamedPattern(prefix + re + suffix, _names(pats));
}

/**
 * Create a new pattern consisting of adjacent input patterns optionally separated by whitespace.
 * @param pats Patterns
 */
function seq(...pats: Pattern[]): NamedPattern {
  return _join(pats, "", "\\s*", "");
}

function one_of(...pats: Pattern[]): NamedPattern {
  return _join(pats, "(?:", "|", ")");
}

function one_of_words(...words: string[]) {
  return word(one_of(...words));
}

function optional(pat: Pattern): NamedPattern {
  return new NamedPattern(`(?:${pat})?`, _names(pat));
}

function word(word: Pattern): NamedPattern {
  return new NamedPattern(`\\b${word}\\b`, _names(word));
}

function named(pat: Pattern, name: string) {
  return new NamedPattern(`(${pat})`, [name, ..._names(pat)]);
}

function recursive_group(
  context_name: string,
  group_name: string,
  body: Pattern
): NamedPattern {
  return new NamedPattern(`(?<${group_name}>${body})`, [
    context_name,
    ..._names(body)
  ]);
}

function recursive_group_reference(group_name: string): Pattern {
  return String.raw`\g<${group_name}>`;
}

const NUMERIC_LITERAL = named(
  "-?\\b(?:(?:0(?:x|X)[0-9a-fA-F]*)|(?:0(?:b|B)[01]*)|(?:(?:[0-9]+\\.?[0-9]*)|(?:\\.[0-9]+))(?:(?:e|E)(?:\\+|-)?[0-9]+)?)\\b",
  "constant.numeric"
);
const BOOLEAN_LITERAL = named(
  one_of_words("true", "false"),
  "constant.language"
);
const STRING_LITERAL = named('"(?:[^\\"]|\\.)*"', "string.quoted.double");
const LITERAL = one_of(NUMERIC_LITERAL, BOOLEAN_LITERAL, STRING_LITERAL);

const IDENTIFIER = "@?\\b[a-zA-Z_][0-9a-zA-Z_]*\\b";
const COMPOUND_IDENTIFIER = new NamedPattern(
  `${IDENTIFIER}(?:\\.${IDENTIFIER})*`
);

const CONSTANT = one_of(LITERAL, COMPOUND_IDENTIFIER);
const NUMERIC_CONSTANT = one_of(NUMERIC_LITERAL, COMPOUND_IDENTIFIER);

const NULLABLE = named("[?]", "punctuation.nullable");

function nullable(pat: Pattern): Pattern {
  return seq(pat, NULLABLE);
}

function angle_brackets(contents: Pattern) {
  const scope = "punctuation.bracket.angle";
  return seq(named("<", scope), contents, named(">", scope));
}

function keyword(keyword: string, name: string = "keyword.control") {
  return word(named(keyword, name));
}

function type_keyword(type_name: string) {
  return keyword(type_name, "storage.type.basic");
}

function separator(sep: string) {
  return named(sep, "punctuation.separator");
}

const primitive_types = [
  "bool",
  "float32",
  "float64",
  "int8",
  "int16",
  "int32",
  "int64",
  "uint8",
  "uint16",
  "uint32",
  "uint64"
];
const handle_types = [
  "bti",
  "channel",
  "debuglog",
  "event",
  "eventpair",
  "fifo",
  "guest",
  "interrupt",
  "job",
  "port",
  "process",
  "profile",
  "resource",
  "socket",
  "thread",
  "timer",
  "vmar",
  "vmo"
];
const PRIMITIVE_TYPE = named(
  one_of_words(...primitive_types),
  "storage.type.basic"
);
const HANDLE_TYPE = named(
  nullable(
    seq(word("handle"), optional(angle_brackets(one_of_words(...handle_types))))
  ),
  "storage.type.basic"
);
const EOL = new NamedPattern("(;)", ["punctuation.terminator"]);
const LIBRARY_NAME = named(COMPOUND_IDENTIFIER, "entity.name.type");
const LOCAL_TYPE = named(IDENTIFIER, "entity.name.type");
const VARIABLE = named(IDENTIFIER, "variable");

const LOOKAHEAD_IDENTIFIER = "(?=[a-zA-Z_@])";

const ATTRIBUTE = seq(
  named(IDENTIFIER, "support.variable"),
  optional(seq("=", STRING_LITERAL))
);

const ATTRIBUTES = block({
  name: "meta.attrbutes",
  begin: String.raw`\[`,
  end: String.raw`\]`,
  patterns: [ATTRIBUTE]
});

const TYPE_CONSTRAINT = seq(
  separator(":"),
  one_of(NUMERIC_LITERAL, COMPOUND_IDENTIFIER)
);

const TYPE_CONSTRUCTOR = recursive_group(
  "entity.name.type",
  "type-constructor",
  seq(
    one_of(
      PRIMITIVE_TYPE,
      HANDLE_TYPE,
      seq(
        one_of(
          type_keyword("vector"),
          type_keyword("array"),
          type_keyword("request"),
          named(word("string"), "storage.type.basic"),
          COMPOUND_IDENTIFIER
        ),
        optional(angle_brackets(recursive_group_reference("type-constructor"))),
        optional(TYPE_CONSTRAINT),
        optional(NULLABLE)
      )
    )
  )
);

// Checks

COMPOUND_IDENTIFIER.assert("foo");
COMPOUND_IDENTIFIER.assert("foo_bar");
COMPOUND_IDENTIFIER.assert("foo.bar.baz");

TYPE_CONSTRUCTOR.assert("string");
TYPE_CONSTRUCTOR.assert("string?");
TYPE_CONSTRUCTOR.assert("string:2");
TYPE_CONSTRUCTOR.assert("string:MAX_LEN");
TYPE_CONSTRUCTOR.assert("string:2?");
TYPE_CONSTRUCTOR.assert("string:MAX_LEN?");

NUMERIC_LITERAL.assert("-42");
NUMERIC_LITERAL.assert("0x12345deadbeef");
NUMERIC_LITERAL.assert("0b101110");

TYPE_CONSTRUCTOR.assert("handle");
TYPE_CONSTRUCTOR.assert("handle?");
TYPE_CONSTRUCTOR.assert("handle<socket>");
TYPE_CONSTRUCTOR.assert("handle<debuglog>?");

TYPE_CONSTRUCTOR.assert("foo.bar.myvectoralias<uint8>:8");

// TODO: support attributes
const tmLanguage: TmLanguage = {
  $schema: tmSchema,
  name: "FIDL",
  scopeName: "source.fidl",
  patterns: [
    include("comments"),

    // Library declaration
    match("meta.library", seq(keyword("library"), LIBRARY_NAME, EOL)),

    // Variations of using
    match("meta.library", seq(keyword("using"), LIBRARY_NAME, EOL)),
    match(
      "meta.library",
      seq(keyword("using"), LIBRARY_NAME, keyword("as"), LOCAL_TYPE, EOL)
    ),
    match(
      "meta.library",
      seq(keyword("using"), LOCAL_TYPE, separator("="), TYPE_CONSTRUCTOR, EOL)
    ),

    // Const declaration
    match(
      "meta.const",
      seq(
        keyword("const"),
        one_of(COMPOUND_IDENTIFIER, PRIMITIVE_TYPE),
        named(IDENTIFIER, "variable.constant"),
        separator("="),
        one_of(CONSTANT, NUMERIC_CONSTANT),
        EOL
      )
    ),

    // Protocols
    block({
      name: "meta.protocol-block",
      begin: seq(keyword("protocol"), LOCAL_TYPE, "{"),
      end: "}",
      patterns: [
        ATTRIBUTES,
        seq(keyword("compose"), named(COMPOUND_IDENTIFIER, "entity.name.type")),
        include("method"),
        include("comments")
      ]
    }),

    // Enums
    block({
      name: "meta.enum-block",
      begin: seq(
        keyword("enum"),
        LOCAL_TYPE,
        optional(seq(separator(":"), TYPE_CONSTRUCTOR)),
        "{"
      ),
      end: "}",
      patterns: [include("enum-member"), include("comments")]
    }),

    // Bits
    block({
      name: "meta.bits-block",
      begin: seq(
        keyword("bits"),
        LOCAL_TYPE,
        optional(seq(separator(":"), TYPE_CONSTRUCTOR)),
        "{"
      ),
      end: "}",
      patterns: [include("enum-member"), include("comments")]
    }),

    // Struct
    block({
      name: "meta.struct-block",
      begin: seq(keyword("struct"), LOCAL_TYPE, "{"),
      end: "}",
      patterns: [include("struct-member"), include("comments")]
    }),

    // Table
    block({
      name: "meta.table-block",
      begin: seq(keyword("table"), LOCAL_TYPE, "{"),
      end: "}",
      patterns: [include("table-member"), include("comments")]
    }),

    // Union and XUnion
    block({
      name: "meta.union-block",
      begin: seq(one_of(keyword("union"), keyword("xunion")), LOCAL_TYPE, "{"),
      end: "}",
      patterns: [include("union-member"), include("comments")]
    })
  ],
  repository: {
    comments: {
      patterns: [
        match("invalid.illegal.stray-comment-end", "\\*/.*\\n"),
        match("comment.line.documentation", "///.*\\n"),
        match("comment.line.double-slash", "//.*\\n")
      ]
    },
    method: {
      patterns: [
        block({
          name: "meta.method",
          begin: seq(named(IDENTIFIER, "entity.name.function")),
          end: seq(optional(seq(keyword("error"), TYPE_CONSTRUCTOR)), EOL),
          patterns: [include("method-arguments"), separator("->")]
        }),
        block({
          name: "meta.method.event",
          begin: seq(
            separator("->"),
            named(IDENTIFIER, "entity.name.function"),
            "[(]"
          ),
          end: seq("[)]", EOL),
          patterns: [include("method-argument")]
        })
      ]
    },
    "method-arguments": {
      patterns: [
        block({
          name: "meta.method.arguments",
          begin: "\\(",
          end: "\\)",
          patterns: [include("method-argument")]
        })
      ]
    },
    "method-argument": {
      patterns: [
        block({
          name: "meta.method.argument",
          begin: LOOKAHEAD_IDENTIFIER,
          end: seq(named(IDENTIFIER, "variable.name"), "(?:(?:,)|(?=\\)))"),
          patterns: [TYPE_CONSTRUCTOR, named(IDENTIFIER, "variable.parameter")]
        })
      ]
    },
    "enum-member": {
      patterns: [
        match(
          "meta.enum.member",
          seq(VARIABLE, separator("="), NUMERIC_CONSTANT, EOL)
        )
      ]
    },
    "struct-member": {
      patterns: [
        match("meta.struct.member", seq(TYPE_CONSTRUCTOR, VARIABLE, EOL)),
        match(
          "meta.struct.member",
          seq(TYPE_CONSTRUCTOR, VARIABLE, separator("="), CONSTANT, EOL)
        )
      ]
    },
    "table-member": {
      patterns: [
        match(
          "meta.table.member",
          seq(NUMERIC_LITERAL, separator(":"), TYPE_CONSTRUCTOR, VARIABLE, EOL)
        ),
        match(
          "meta.table.reserved",
          seq(NUMERIC_LITERAL, separator(":"), keyword("reserved"), EOL)
        )
      ]
    },
    "union-member": {
      patterns: [
        match(
          "meta.union.member",
          seq(NUMERIC_LITERAL, separator(":"), TYPE_CONSTRUCTOR, VARIABLE, EOL)
        ),
        match(
          "meta.union.member.reserved",
          seq(NUMERIC_LITERAL, separator(":"), keyword("reserved"), EOL)
        )
      ]
    },
    attributes: {
      patterns: []
    }
  }
};

console.log(JSON.stringify(tmLanguage, null, "    "));