syntax/cml.ts

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// This generates a language definition file because writing it by hand is too hard.

// Format of language definition JSON
const tmSchema =
  'https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json';
type TmCaptures = { [k: string]: { name: string } };
type TmIncludePattern = { include: string };
type TmMatchPattern = {
  name?: string;
  match: string;
  captures?: TmCaptures;
};
type TmBlockPattern = {
  name?: string;
  begin: string;
  beginCaptures?: TmCaptures;
  end: string;
  endCaptures?: TmCaptures;
  patterns: TmPattern[];
};
type TmPattern = TmIncludePattern | TmMatchPattern | TmBlockPattern;
type TmLanguage = {
  $schema: string;
  name: string;
  scopeName: string;
  patterns: TmPattern[];
  repository: {
    [key: string]: { patterns: TmPattern[] };
  };
};

class NamedPattern {
  readonly re: string;
  readonly names: string[];
  constructor(re: string, names?: string[]) {
    this.re = re;
    this.names = names || [];
    // Hack to only validate if there are no unresolved recursive groups
    const recursiveRE = /\\g<(.*?)>/g;
    let match;
    while ((match = recursiveRE.exec(re)) !== null) {
      if (!re.includes(`(?<${match[1]}>`)) {
        // Group definition not found, can't validate yet.
        return;
      }
    }
    this.validate();
  }

  validate() {
    // TODO(fxbug.dev/119360): reenable this
    // const groupCount = new OnigRegExp(this.re + '|').searchSync('')!.length - 1;
    // if (groupCount !== this.names.length) {
    //   throw new Error(
    //     `Found ${groupCount} but expected ${this.names.length} groups in ${this.re}`
    //   );
    // }
  }

  toString() {
    return this.re;
  }

  assert(_s: string) {
    // TODO(fxbug.dev/119360): reenable this
    // const re = new OnigRegExp(this.re);
    // const m = re.searchSync(s);
    // if (!m) {
    //   throw Error(
    //     `${JSON.stringify(s)} did not match pattern ${JSON.stringify(this.re)}`
    //   );
    // }
    // const c = m[0];
    // if (c.index !== 0 || c.start !== 0 || c.length !== s.length) {
    //   throw Error(
    //     `${JSON.stringify(s)} did not fully match pattern ${JSON.stringify(
    //       this.re
    //     )}, only matched ${JSON.stringify(m[0])}`
    //   );
    // }
  }
}

/**
 * Create a dictionary of capture group names: <captureIdx, nameId>
 * @param pattern input pattern
 * @returns dictionary
 */
function getCaptures(pattern: Pattern): TmCaptures {
  const captures: { [k: string]: { name: string } } = {};
  const names = getPatternNames(pattern);
  for (let i = 0; i < names.length; i++) {
    captures[`${i + 1}`] = { name: names[i] };
  }
  return captures;
}

/**
 * Create TmIncludePattern from name identifier
 * @param name identifier
 * @returns TmIncludePattern
 */
function include(name: string): TmIncludePattern {
  return { include: `#${name}` };
}

/**
 * Create a match pattern for the output json syntax.
 * @param name
 * @param pattern
 * @returns pattern struct for the output json.
 */
function match(name: string, pattern: Pattern): TmMatchPattern {
  return {
    name: `${name}.cml`,
    match: pattern.toString(),
    captures: getCaptures(pattern),
  };
}

/**
 * Create a match pattern with no name for the output json syntax.
 * @param pattern
 * @returns pattern struct for the output json.
 */
function anonMatch(pattern: Pattern | TmPattern): TmPattern {
  if (pattern instanceof String) {
    return {
      match: pattern.toString(),
    };
  }
  if (pattern instanceof NamedPattern) {
    return {
      match: pattern.toString(),
      captures: getCaptures(pattern),
    };
  }
  return pattern;
}

/**
 * Create block pattern for the output json syntax.
 * @param args
 * @returns pattern struct for the output json.
 */
function block(args: {
  name: string;
  begin: Pattern;
  end: Pattern;
  patterns: Array<Pattern | TmPattern>;
}): TmBlockPattern {
  const tmPatterns: TmPattern[] = args.patterns.map(anonMatch);
  return {
    name: `${args.name}.cml`,
    begin: args.begin.toString(),
    beginCaptures: getCaptures(args.begin),
    end: args.end.toString(),
    endCaptures: getCaptures(args.end),
    patterns: tmPatterns,
  };
}

/**
 * Creates a key-value array dictionary, k: [{}]
 * @param args
 * @returns pattern struct for the output json.
 */
function kvArrayDictionary(args: {
  name: string;
  begin: Pattern;
  inner: Array<Pattern | TmPattern>;
}): TmBlockPattern {
  return keyValue(
    args.name,
    [
      ArrayBlock([DictionaryBlock(args.inner)])
    ],
  );
}

/**
 * Creates a key-value dictionary, k: {}
 * @param args
 * @returns pattern struct for the output json.
 */
function kvDictionary(args: {
  name: string;
  begin: Pattern;
  inner: Array<Pattern | TmPattern>;
}): TmBlockPattern {
  return keyValue(
    args.name,
    [DictionaryBlock(args.inner)],
  );
}

/**
 * Creates an array block
 * @param pattern used for every array element
 * @param valid is an array block a valid type?
 * @returns pattern struct for the output json.
 */
function ArrayBlock(pattern?: Array<Pattern | TmPattern>, valid?: boolean): TmBlockPattern {
  if (pattern === undefined) {
    pattern = [patterns.anyString()];
  }
  valid = valid ?? true;
  return block({
    name: 'meta.meta-array-block',
    begin: patterns.named('\\[', valid ? 'punctuation.definition.array.begin' : 'invalid.illegal'),
    end: patterns.named('\\]', valid ? 'punctuation.definition.array.end' : 'invalid.illegal'),
    patterns: [
      ...pattern,
      include('comments'),
      //
      patterns.named('(?<=,)[ ]*,', 'invalid.illegal'),
      patterns.separator(','),
      patterns.named('\\.', 'invalid.illegal'),
      patterns.anyString('invalid.illegal'),
      patterns.identifier('invalid.illegal'),
      patterns.integer('invalid.illegal'),
      patterns.boolValue('invalid.illegal'),
      include('meta-invalid-array-block'),
      include('meta-invalid-dictionary-block'),
    ],
  });
}

/**
 * Creates a dictionary, {k:v,}
 * @param pattern pattern for the key-value pair
 * @param valid is the dictionary a valid value type?
 * @returns
 */
function DictionaryBlock(pattern: Array<Pattern | TmPattern>, valid?: boolean): TmBlockPattern {
  valid = valid ?? true;
  return block({
    name: 'meta.meta-dictionary-block',
    begin: patterns.named('\\{', valid ? 'punctuation.definition.dictionary.begin' : 'invalid.illegal'),
    end: patterns.named('\\}', valid ? 'punctuation.definition.dictionary.end' : 'invalid.illegal'),
    patterns: [
      ...pattern,
      include('comments'),

      include('meta-invalid-key-value-block'),
      patterns.named(',', 'invalid.illegal'),
      patterns.named('\\.', 'invalid.illegal'),
      patterns.anyString('invalid.illegal'),
      patterns.identifier('invalid.illegal'),
      patterns.integer('invalid.illegal'),
      patterns.boolValue('invalid.illegal'),
      include('meta-invalid-array-block'),
      include('meta-invalid-dictionary-block'),
    ],
  });
}

/**
 * Creates a key-value block, k:v
 * @param name block name
 * @param pattern array pattern for values
 * @param key pattern for key
 * @returns pattern struct for the output json.
 */
function keyValue(name: string,
  pattern?: Array<Pattern | TmPattern>,
  key?: NamedPattern): TmBlockPattern {
  if (pattern === undefined) {
    pattern = [patterns.anyString()];
  }
  let keyPattern = key ?? patterns.oneFrom(
    patterns.named(name, 'keyword.control'),
    patterns.oneString(name, 'keyword.control')
  );
  return block({
    name: `meta.meta-${name}-block`,
    begin: patterns.seq(keyPattern, '\\:'),
    end: patterns.oneFrom(patterns.separator(','), '(?=})'),
    patterns: [
      ...pattern,

      patterns.named('\\.', 'invalid.illegal'),
      patterns.anyString('invalid.illegal'),
      patterns.identifier('invalid.illegal'),
      patterns.integer('invalid.illegal'),
      patterns.boolValue('invalid.illegal'),
      include('meta-invalid-array-block'),
      include('meta-invalid-dictionary-block'),
    ],
  });
}

///////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////

type Pattern = NamedPattern | String;

/** Return the names of the groups in the pattern. */
function getPatternNames(pat: Pattern | Pattern[]): string[] {
  if (pat instanceof Array) {
    return pat.map(getPatternNames).reduce((prev, names) => [...(prev || []), ...names]);
  }
  if (pat instanceof NamedPattern) {
    return pat.names;
  } else {
    return [];
  }
}

/**
 * Utility to create a NamedPattern by concatenating an input pattern array. Output =
 * [prefix + (pat + separator?)* + suffix]
 * @param pats input pattern array.
 * @param prefix
 * @param separator used to concatenate input patterns together.
 * @param suffix
 * @returns the resulting NamedPattern.
 */
function joinPattern(
  pats: Pattern[],
  prefix: string,
  separator: string,
  suffix: string
): NamedPattern {
  const re = pats.map((p) => p.toString()).join(separator);
  return new NamedPattern(prefix + re + suffix, getPatternNames(pats));
}

// NamedPattern Factory.
var patterns = {
  /**
   * Create a new pattern consisting of adjacent input patterns optionally separated by whitespaces.
   * @param pats Patterns
   */
  seq: function (...pats: Pattern[]): NamedPattern {
    return joinPattern(pats, '', '\\s*', '');
  },

  /**
   * Create a word NamedPattern match by searching word boundaries.
   * @param word input pattern
   * @returns NamedPattern.
   */
  word: function (word: Pattern): NamedPattern {
    return new NamedPattern(`\\b${word}\\b`, getPatternNames(word));
  },

  /**
   * Create a double quoted string pattern matcher.
   * @param word desired word pattern.
   * @param type pattern name.
   * @returns NamedPattern.
   */
  oneString: function (word: Pattern, type?: string): NamedPattern {
    return patterns.named(`"${word}"`, type ?? 'string.quoted.double');
  },

  /**
   * Create a non-capturing NamedPattern group that matches one item from the pattern input vector.
   * @param pats pattern input vector
   * @returns NamedPattern.
   */
  oneFrom: function (...pats: Pattern[]): NamedPattern {
    return joinPattern(pats, '(?:', '|', ')');
  },

  /**
   * Create a double quoted string pattern matcher from a word array.
   * @param words that can match pattern.
   * @returns NamedPattern.
   */
  oneStringFrom: function (...words: string[]) {
    return patterns.oneString(patterns.oneFrom(...words));
  },

  /**
   * Match true or false for boolean values
   * @param type overrides the default type.
   * @returns NamedPattern.
   */
  boolValue: function (type?: string) {
    return patterns.oneFrom(
      patterns.named('\\btrue\\b', type ?? 'constant.language.true'),
      patterns.named('\\bfalse\\b', type ?? 'constant.language.false')
    );
  },

  /**
   * Match integer numbers
   * @param type overrides the default type.
   * @returns NamedPattern.
   */
  integer: function (type?: string) {
    return patterns.named('\\b[0-9]+\\b', type ?? 'storage.type.int');
  },

  /**
   * Match identifiers
   * @param type overrides the default type.
   * @returns NamedPattern.
   */
  identifier: function (type: string) {
    return patterns.named('\\b[a-zA-Z_][0-9a-zA-Z_]*\\b', type);
  },

  /**
   * Match any string
   * @param type overrides the default type.
   * @returns NamedPattern.
   */
  anyString: function (type?: string) {
    type = type ?? 'string.quoted.double';
    return patterns.named('"(?:[^\\"]|\\.)*"', type);
  },

  /**
   * Match any component name
   * @param type overrides the default type.
   * @returns NamedPattern.
   */
  componentName: function (type?: string) {
    type = type ?? 'string.quoted.double';
    return patterns.named('"[0-9a-z_\\-\\.]*"', type);
  },

  /**
   * Regular expression for references, e.g. #<child-name>, #<collection-name>
   * @returns Pattern
   */
  nameReference: function () {
    return patterns.named('"#[0-9a-z_\\-\\.]+"', 'string.quoted.double');
  },

  /**
   * Create a NamedPattern.
   * @param pattern
   * @param name
   * @returns NamedPattern
   */
  named: function (pattern: Pattern, name: string) {
    return new NamedPattern(`(${pattern})`, [name, ...getPatternNames(pattern)]);
  },

  /**
   * Create a NamedPattern for separators.
   * @param separator
   * @returns
   */
  separator: function (separator: string) {
    return patterns.named(separator, 'punctuation.separator');
  },

  /**
   * Create a word NamedPattern from a keyword string.
   * @param keyword string
   * @param name keyword identifier
   * @returns NamedPattern
   */
  keyword: function (keyword: string, name: string = 'keyword.control') {
    return patterns.word(patterns.named(keyword, name));
  },
};

/**
 * Value or Array block
 * @param pattern for a single value or each element of an array, default pattern = anyString()
 * @returns An array of patterns.
 */
function ValueOrArray(pattern: Array<Pattern | TmPattern> = [patterns.anyString()]): Array<Pattern | TmPattern> {
  return [...pattern, ArrayBlock(pattern)];
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// CML language description.
///////////////////////////////////////////////////////////////////////////////////////////////////

const tmLanguage: TmLanguage = {
  $schema: tmSchema,
  name: 'CML',
  scopeName: 'source.cml',

  patterns: [
    include('comments'),

    DictionaryBlock([
      keyValue('include', [ArrayBlock()]),
      include('meta-program-block'),
      include('meta-children-block'),
      include('meta-collections-block'),
      include('meta-environments-block'),
      include('meta-capabilities-block'),
      include('meta-use-block'),
      include('meta-expose-block'),
      include('meta-offer-block'),
      keyValue('facets', [
        include('meta-valid-array-block'),
        include('meta-valid-dictionary-block'),
      ]),
      keyValue('config', [
        include('meta-valid-array-block'),
        include('meta-valid-dictionary-block'),
      ]),
    ]),
  ],

  repository: {
    'meta-program-block': {
      patterns: [
        kvDictionary({
          name: 'program',
          begin: patterns.keyword('program'),
          inner: [
            keyValue('runner'),
            keyValue('binary'),
            keyValue('args', [ArrayBlock()]),
            // In case of using a runner other than elf.
            include('meta-valid-key-value-block'),
          ],
        }),
      ],
    },

    'meta-children-block': {
      patterns: [
        kvArrayDictionary({
          name: 'children',
          begin: patterns.keyword('children'),
          inner: [
            keyValue('name', [patterns.componentName()]),
            keyValue('url'),
            keyValue('startup', [patterns.oneStringFrom('lazy', 'eager')]),
            keyValue('on_terminate', [patterns.oneStringFrom('none', 'reboot')]),
            keyValue('environment'),
          ],
        }),
      ],
    },

    'meta-collections-block': {
      patterns: [
        kvArrayDictionary({
          name: 'collections',
          begin: patterns.keyword('collections'),
          inner: [
            keyValue('name', [patterns.componentName()]),
            keyValue('durability', [patterns.oneStringFrom('transient', 'single_run')]),
            keyValue('environment'),
            keyValue('allowed_offers', [patterns.oneStringFrom('static_only', 'static_and_dynamic')]),
            keyValue('allow_long_names', [patterns.boolValue()]),
            keyValue('persistent_storage', [patterns.boolValue()]),
          ],
        }),
      ],
    },

    'meta-environments-block': {
      patterns: [
        kvArrayDictionary({
          name: 'environments',
          begin: patterns.keyword('environments'),
          inner: [
            keyValue('name', [patterns.componentName()]),
            keyValue('extends', [patterns.oneStringFrom('realm', 'none')]),
            kvArrayDictionary({
              name: 'runners',
              begin: patterns.keyword('runners'),
              inner: [
                keyValue('runner'),
                keyValue('from'),
                keyValue('as'),
              ],
            }),
            kvArrayDictionary({
              name: 'resolvers',
              begin: patterns.keyword('resolvers'),
              inner: [
                keyValue('resolver'),
                keyValue('from'),
                keyValue('scheme'),
              ],
            }),
            kvArrayDictionary({
              name: 'debug',
              begin: patterns.keyword('debug'),
              inner: [
                keyValue('protocol', [ArrayBlock(), patterns.anyString()]),
                keyValue('from'),
                keyValue('as'),
              ],
            }),
            keyValue('stop_timeout_ms', [patterns.integer()]),
          ],
        }),
      ],
    },

    'meta-capabilities-block': {
      patterns: [
        kvArrayDictionary({
          name: 'capabilities',
          begin: patterns.keyword('capabilities'),
          inner: [
            keyValue('service', [ArrayBlock(), patterns.anyString()]),
            keyValue('protocol', [ArrayBlock(), patterns.anyString()]),
            keyValue('directory'),
            keyValue('storage'),
            keyValue('runner'),
            keyValue('resolver'),
            keyValue('event'),
            keyValue('event_stream', [ArrayBlock(), patterns.anyString()]),
            keyValue('path'),
            keyValue('rights', [ArrayBlock(), patterns.anyString()]),
            keyValue('from'),
            keyValue('backing_dir'),
            keyValue('subdir'),
            keyValue('storage_id', [patterns.oneStringFrom('static_instance_id', 'static_instance_id_or_moniker')]),
          ],
        }),
      ],
    },

    'meta-use-block': {
      patterns: [
        kvArrayDictionary({
          name: 'use',
          begin: patterns.keyword('use'),
          inner: [
            keyValue('service', [ArrayBlock(), patterns.anyString()]),
            keyValue('protocol', [ArrayBlock(), patterns.anyString()]),
            keyValue('directory'),
            keyValue('storage'),
            keyValue('event', [ArrayBlock(), patterns.anyString()]),
            keyValue('event_stream_deprecated'),
            keyValue('event_stream', [ArrayBlock(), patterns.anyString()]),
            keyValue('from', [
              patterns.oneStringFrom('self', 'framework', 'parent', 'debug'),
              patterns.nameReference(),
            ]),
            keyValue('path'),
            keyValue('rights', [ArrayBlock(), patterns.anyString()]),
            keyValue('subdir'),
            keyValue('as'),
            keyValue('scope', [ArrayBlock(), patterns.anyString()]),
            //TODO(fxbug.dev/109399): Narrow down the types of valid objects once documentation is
            //provided in fxbug.dev/96705.
            keyValue('filter', [
              include('meta-valid-array-block'),
              include('meta-valid-dictionary-block'),
            ]),
            keyValue('subscriptions'),
            keyValue('dependency', [patterns.oneStringFrom('strong', 'weak_for_migration', 'weak')]),
            keyValue('availability', [patterns.oneStringFrom('required', 'optional')]),
          ],
        }),
      ],
    },

    'meta-expose-block': {
      patterns: [
        kvArrayDictionary({
          name: 'expose',
          begin: patterns.keyword('expose'),
          inner: [
            keyValue('service', [ArrayBlock(), patterns.anyString()]),
            keyValue('protocol', [ArrayBlock(), patterns.anyString()]),
            keyValue('directory', [ArrayBlock(), patterns.anyString()]),
            keyValue('runner', [ArrayBlock(), patterns.anyString()]),
            keyValue('resolver', [ArrayBlock(), patterns.anyString()]),
            keyValue('from', [
              patterns.oneStringFrom('self', 'framework'),
              patterns.nameReference(),
            ]),
            keyValue('as'),
            keyValue('to', [
              patterns.oneStringFrom('parent', 'framework'),
            ]),
            keyValue('rights', [ArrayBlock(), patterns.anyString()]),
            keyValue('subdir'),
            keyValue('event_stream', [ArrayBlock(), patterns.anyString()]),
            keyValue('scope', [ArrayBlock(), patterns.anyString()]),
          ],
        }),
      ],
    },

    'meta-offer-block': {
      patterns: [
        kvArrayDictionary({
          name: 'offer',
          begin: patterns.keyword('offer'),
          inner: [
            keyValue('service', [ArrayBlock(), patterns.anyString()]),
            keyValue('protocol', [ArrayBlock(), patterns.anyString()]),
            keyValue('directory', [ArrayBlock(), patterns.anyString()]),
            keyValue('runner', [ArrayBlock(), patterns.anyString()]),
            keyValue('resolver', [ArrayBlock(), patterns.anyString()]),
            keyValue('storage', [ArrayBlock(), patterns.anyString()]),
            keyValue('event', [ArrayBlock(), patterns.anyString()]),
            keyValue('from', ValueOrArray([
              patterns.oneStringFrom('parent', 'self', 'framework', 'void'),
              patterns.nameReference(),
            ])),
            keyValue('to', ValueOrArray([
              patterns.oneStringFrom('parent', 'framework'),
              patterns.nameReference(),
            ])),
            keyValue('as'),
            keyValue('dependency', [patterns.oneStringFrom('strong', 'weak_for_migration', 'weak')]),
            keyValue('rights', [ArrayBlock(), patterns.anyString()]),
            keyValue('subdir'),
            //TODO(fxbug.dev/109399): Narrow down the types of valid objects once documentation is
            //provided in fxbug.dev/96705.
            keyValue('filter', [
              include('meta-valid-array-block'),
              include('meta-valid-dictionary-block'),
            ]),
            keyValue('event_stream', [ArrayBlock(), patterns.anyString()]),
            keyValue('scope', [ArrayBlock(), patterns.anyString()]),
            keyValue('availability', [patterns.oneStringFrom('required', 'optional', 'same_as_target')]),
            keyValue('source_availability'),
          ],
        }),
      ],
    },

    ///////////////////////////////////////////////////////////////////////////////////////////////
    // General valid/invalid blocks
    ///////////////////////////////////////////////////////////////////////////////////////////////
    'meta-invalid-array-block': {
      patterns: [
        ArrayBlock([
          patterns.anyString(),
          patterns.integer(),
          patterns.boolValue(),
          include('meta-valid-array-block'),
          include('meta-valid-dictionary-block'),
        ], false)
      ],
    },

    'meta-valid-array-block': {
      patterns: [
        ArrayBlock([
          patterns.anyString(),
          patterns.integer(),
          patterns.boolValue(),
          include('meta-valid-array-block'),
          include('meta-valid-dictionary-block'),
        ], true)
      ],
    },

    'meta-invalid-key-value-block': {
      patterns: [
        keyValue(
          'invalid-key-value',
          [
            patterns.anyString(),
            patterns.integer(),
            patterns.boolValue(),
            include('meta-valid-array-block'),
            include('meta-valid-dictionary-block'),
          ],
          patterns.oneFrom(
            patterns.identifier('invalid.illegal'),
            patterns.anyString('invalid.illegal'),
          )
        )
      ],
    },

    'meta-valid-key-value-block': {
      patterns: [
        keyValue(
          'any-key-value',
          [
            patterns.anyString(),
            patterns.integer(),
            patterns.boolValue(),
            include('meta-valid-array-block'),
            include('meta-valid-dictionary-block'),
          ],
          patterns.oneFrom(
            patterns.identifier('keyword.control'),
            patterns.anyString('keyword.control'),
          )
        )
      ],
    },

    'meta-invalid-dictionary-block': {
      patterns: [
        DictionaryBlock([
          include('meta-valid-key-value-block'),
        ], false)
      ],
    },

    'meta-valid-dictionary-block': {
      patterns: [
        DictionaryBlock([
          include('meta-valid-key-value-block'),
        ], true)
      ],
    },

    ///////////////////////////////////////////////////////////////////////////////////////////////
    // Comments
    ///////////////////////////////////////////////////////////////////////////////////////////////
    comments: {
      patterns: [
        match('invalid.illegal.stray-comment-end', '\\*/.*\\n'),
        match('comment.line.documentation', '///.*\\n'),
        match('comment.line.double-slash', '//.*\\n'),
      ],
    },

  },
};

export default tmLanguage;