tools/fidl/scripts/fidl_converter.py

#!/usr/bin/env fuchsia-vendored-python
# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import os
import subprocess
import sys
from pathlib import Path
from multiprocessing.dummy import Pool

FUCHSIA_DIR = Path(os.environ["FUCHSIA_DIR"])


def main(args):
    """Converts as many FIDL files as possible by invoking `fx build`, then
    running through a number of strategies to match the converted file output.
    This script assumes that the fidlc has been instrumented such that it
    produces a `.fidl.new` type output for each input `.fidl` file.
    """

    if args.dryrun:
        print("\nDRY RUN!")
    else:
        print("\nREAL RUN!")

    if args.rebuild:
        # Note: This includes all of the "kitchen_sink" and "buildbot/core", but
        # unfortunately there are still some build targets that are not included
        # in these catch-all targets.  Those are manually listed at th marked
        # point, but will likely change in the future (ie, the targets could be
        # added to the catch-alls, or removed completely.  Additionally, other
        # targets that are missed by the catch-alls could be added as well).
        print("\nSETTING ENVIRONMENT...")
        subprocess.check_call(
            [
                "fx",
                "--dir=out/default",
                "set",
                "workstation_eng.x64",
                "--with=//bundles/fidl:tests",
                "--with=//bundles/kitchen_sink",
                "--with=//bundles/tests",
                "--with=//bundles/buildbot/core",
                "--with=//sdk/fidl/fuchsia.firebase.messaging:fuchsia.firebase.messaging",
                "--with=//sdk/fidl/fuchsia.metricbroker:fuchsia.metricbroker",
                "--with=//sdk/fidl/fuchsia.process:fuchsia.process",
                "--with=//sdk/fidl/fuchsia.process.init:fuchsia.process.init",
                "--with=//sdk/fidl/fuchsia.terminal:fuchsia.terminal",
                "--with=//src/camera/bin/factory:fuchsia.factory.camera",
                "--with=//src/media/vnext/fidl/fuchsia.audio:fuchsia.audio",
                "--with=//src/media/vnext/fidl/fuchsia.media2:fuchsia.media2",
                "--with=//src/media/vnext/fidl/fuchsia.mediastreams:fuchsia.mediastreams",
                "--with=//src/media/vnext/fidl/fuchsia.mem2:fuchsia.mem2",
                "--cargo-toml-gen",
            ],
            stdout=sys.stdout,
        )

        print("\nBUILDING...")
        subprocess.check_call(["fx", "build"], stdout=sys.stdout)

    print("\nMATCHING... (takes a few min)")
    converted, unconverted = match_converted_files()
    ready = len(converted)
    total = ready + len(unconverted)

    print("\nCOULD NOT MATCH THE FOLLOWING FILES:")
    for u in unconverted:
        print(u)

    if args.dryrun:
        print("MATCHED %s OF %d FIDL FILES" % (ready, total))
    else:
        for old, new in converted.items():
            try:
                subprocess.check_call(["cp", "-fT", new, old])
            except subprocess.CalledProcessError as e:
                print(e.output)
        print("SUCCESSFULLY CONVERTED %s OF %d FIDL FILES" % (ready, total))


def match_converted_files():
    """Attempts to match each source FIDL file with its converted `.fidl.new`
    counterpart in the `out/default` directory.  Returns a tuple, with the first
    value being a map of old_file -> converted_file, and the second being a list
    of old files that could not be converted.
    """

    converted = {}
    unconverted = []

    # Match each source file with its converted copy.
    for path in FUCHSIA_DIR.rglob("*.fidl"):
        if not path.is_file():
            continue
        old_syntax_path = path.relative_to(FUCHSIA_DIR)

        # These are not fuchsia.git source files - ignore them.
        if str(old_syntax_path).startswith("out/"):
            continue
        if str(old_syntax_path).startswith("prebuilt/"):
            continue
        if str(old_syntax_path).startswith("third_party/"):
            continue
        if str(old_syntax_path).startswith("vendor/"):
            continue

        new_syntax_path = (
            FUCHSIA_DIR / "out/default/fidling/gen" / old_syntax_path
        )
        new_syntax_path = new_syntax_path.with_suffix(".fidl.new")

        # The default case: the file's output location matches its location in
        # the source, save a slight redirection to the output directory.
        if new_syntax_path.exists():
            converted[str(old_syntax_path)] = str(new_syntax_path)
            continue

        # Sometimes, a file stored in `foo/fidl/my.fidl` is output into
        # a directory called `foo` rather than `foo/fidl`.  Check for such
        # cases.
        new_syntax_path = "/".join(str(new_syntax_path).rsplit("/fidl/", 1))
        if Path(new_syntax_path).exists():
            converted[str(old_syntax_path)] = str(new_syntax_path)
            continue

        unconverted.append(str(old_syntax_path))

    # But wait!  There's still a couple of things we can try for all of the
    # old syntax FIDL files we've not been able to successfully pair with a
    # converted copy.  First, we'll try using the `fx gn outputs` command on
    # each of the old files, in order to find outputs written to unorthodox
    # output directory paths.  Because this command can take a long time, we'll
    # use a thread pool to parallelize the invocations.
    pool = Pool(16)
    for matched in pool.map(match_converted_file, unconverted):
        if matched is not None:
            converted[matched[0]] = matched[1]
            unconverted.remove(matched[0])

    # One more thing we can try: just attempt to run fidlc on each of the
    # remaining files.  This will fail for FIDL libraries that import
    # dependencies via the `using` command, or are defined over multiple source
    # files, but it will at least cover us for simple test files (DIFL, etc).
    for matched in pool.map(try_convert_fidl_file, unconverted):
        if matched is not None:
            converted[matched[0]] = matched[1]
            unconverted.remove(matched[0])

    # Whatever is left at this point truly, really needs to be converted by
    # hand.
    return converted, unconverted


def match_converted_file(old_syntax_path_str):
    """This function uses `fx gn outputs` to search for any output files located
    in unconventionally named outputs directories. It will return None on
    failure, and a (old_file_path, converted_file_path) tuple on success.
    """

    new_syntax_path = Path(old_syntax_path_str).with_suffix(".fidl.new")
    new_file_name = str(new_syntax_path.name)
    result = subprocess.run(
        ["fx", "gn", "outputs", "out/default", old_syntax_path_str],
        capture_output=True,
    )

    for line in result.stdout.decode("utf-8").splitlines():
        stripped = str(line.strip())
        if stripped.endswith(new_file_name):
            # print(' * ' + old_syntax_path_str + ' [MATCHED]')
            return old_syntax_path_str, "out/default/" + stripped

    # print(' * ' + old_syntax_path_str + ' [NOT FOUND]')
    return None


def try_convert_fidl_file(old_syntax_path_str):
    """This function will successfully convert any FIDL library that does not
    have any `using` imports and is defined in a single source file. It will
    return None on failure, and a (old_file_path, converted_file_path) tuple on
    success.
    """

    tmpdir = Path.home() / "tmp/fidl-migration"
    new_syntax_path = tmpdir / old_syntax_path_str
    new_syntax_path = new_syntax_path.with_suffix(".fidl.new")
    new_syntax_dir = new_syntax_path.parent
    subprocess.check_call(["mkdir", "-p", new_syntax_dir], stdout=sys.stdout)

    # Try to convert the file assuming that it is a standalone library with no
    # imports.  If it works, great.  If not, return None, so that we may record
    # this file as unconvertable.
    fidlc = FUCHSIA_DIR / "out/default/host_x64/exe.unstripped/fidlc"
    old_file = FUCHSIA_DIR / Path(old_syntax_path_str)
    try:
        subprocess.check_call(
            [
                fidlc,
                "--experimental",
                "old_syntax_only",
                "--convert-syntax",
                new_syntax_dir,
                "--files",
                old_file,
            ],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
        )
    except subprocess.CalledProcessError as e:
        # print(' * ' + old_syntax_path_str + ' [ERRORED]')
        return None

    # print(' * ' + old_syntax_path_str + ' [MATCHED]')
    return old_syntax_path_str, str(new_syntax_path)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Copies files converted by fidlc via a flag-enabled mode back into the source tree."
    )
    parser.add_argument(
        "--dry-run",
        dest="dryrun",
        action=argparse.BooleanOptionalAction,
        default=False,
    )
    parser.set_defaults(dryrun=True)
    parser.add_argument(
        "--rebuild",
        dest="rebuild",
        action=argparse.BooleanOptionalAction,
        default=False,
    )
    parser.set_defaults(rebuild=True)
    main(parser.parse_args())