scripts/jiri/find_unused.py

#!/usr/bin/env python3.8
# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Attempts to find unused jiri projects.
For each project in a given jiri manifest file, do the following:
1. Attempt to break incoming deps to this project by renaming the path in which
   it's installed.
2. Try to `fx gen`, as a smoke test for whether this is unused. Log the result.
3. If #2 was successful, upload a change to remove the project and log the URL.

The data is collected in a csv.
Users are encouraged to follow up on all changes uploaded and see if they
passed CQ, then add another column with said result. Changes that didn't pass
CQ probably indicate that the project is used.

Example usage:
$ fx set ...
$ scripts/jiri/find_unused.py \
--jiri_manifest integration/third_party/flower
--csv output.csv
"""

import argparse
import os
import re
import subprocess
import sys
import xml.etree.ElementTree as ET


def run_returncode(*command, **kwargs):
    return subprocess.run(
        command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
        **kwargs).returncode


def check_output(*command, **kwargs):
    try:
        return subprocess.check_output(
            command, stderr=subprocess.STDOUT, encoding="utf8", **kwargs)
    except subprocess.CalledProcessError as e:
        print("Failed: " + " ".join(command), file=sys.stderr)
        print(e.output, file=sys.stderr)
        raise e


def main():
    parser = argparse.ArgumentParser(
        description="Collects data useful in trimming unused jiri projects")
    parser.add_argument(
        "--jiri_manifest", help="Input jiri manifest file", required=True)
    parser.add_argument(
        "--csv",
        help="Output csv file",
        type=argparse.FileType('w'),
        required=True)
    args = parser.parse_args()

    URL = re.compile("https:\/\/\S*")

    # Check that integration repository is clean
    manifest_dir = os.path.dirname(args.jiri_manifest)

    def check_git_command(*command):
        try:
            check_output("git", *command, cwd=manifest_dir)
        except subprocess.CalledProcessError as e:
            print("Failed: git " + " ".join(command), file=sys.stderr)
            print(e.output, file=sys.stderr)
            raise e

    check_git_command("status", "--porcelain")

    manifest_contents = open(args.jiri_manifest).read()
    # Parse jiri manifest
    manifest = ET.parse(args.jiri_manifest)
    root = manifest.getroot()
    projects = root.find("projects")
    print(f"Found {len(projects)} projects.")

    for project in projects.findall("project"):
        path = project.get("path")
        print(f"Checking {path}...")
        if not os.path.exists(path):
            print(f"{path} not in your checkout!")
            args.csv.write(f"{path}\n")
            continue

        # Break path and see if GN notices
        os.rename(path, path + "_broken_by_find_unused_py")
        gen_result = run_returncode("fx", "gen")
        os.rename(path + "_broken_by_find_unused_py", path)
        if gen_result != 0:
            args.csv.write(f"{path},false\n")
            print(f"{path} is used!")
            continue

        # Remove project and send to CQ
        # We could simply remove the tree element and write to the file, but
        # this will destroy any comments and formatting.
        # Instead, mess around with regular expressions to delete in-place.
        # Perhaps a more robust solution is to first ensure that the jiri
        # manifest is formatted in such a way that we can recreate, such as by
        # parsing it, serializing it to bytes, and comparing it to the original
        # file bytes.
        # This option is left as an exercise to the reader.
        new_manifest_contents = re.sub(
            "^\s*<project [^\>]*?path\s*=\s*\"" + re.escape(path) +
            "\"[\S\s]*?\/>\n",
            "",
            manifest_contents,
            flags=re.MULTILINE)
        open(args.jiri_manifest, 'w').write(new_manifest_contents)
        check_git_command(
            "commit", "-a", "-m", f"[jiri] Check if {path} is unused")
        print(f"Sending {path} removal to CQ... ", end='')
        upload_out = check_output(
            "jiri", "upload", "-l", "Commit-Queue+1", cwd=manifest_dir)
        url = URL.search(upload_out)[0]
        print(url)
        args.csv.write(f"{path},true,{url}\n")

        # Undo project removal
        check_git_command("checkout", "HEAD^")


if __name__ == "__main__":
    sys.exit(main())