build/packages/generate_sdk_package_content_checklist.py

#!/usr/bin/env fuchsia-vendored-python
# Copyright 2023 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Generates a content checklist file for inclusion in the IDK by reading the passed
package manifest, files (with specified dispositions), and reference to compare
with. For use in sdk_fuchsia_package() template."""

import argparse
import collections
import difflib
import json
import os
import subprocess
import sys
import tempfile


def get_meta_far_contents(ffx_bin, far_bin, meta_far_source_path):
    """
    Takes a path to a meta far file, and returns a list
    of tuples of form: (file_path, content_hash)
    """
    # Extract file paths from meta.far
    meta_far_paths_and_merkles = []
    meta_far_list_result = subprocess.run(
        [far_bin, "list", f"--archive={meta_far_source_path}"],
        stdout=subprocess.PIPE,
        text=True,
    )
    meta_far_file_paths = meta_far_list_result.stdout.split("\n")

    # tempdir used for isolating ffx
    with tempfile.TemporaryDirectory() as tmpdir:
        # Extract contents of file paths, and calculate content hash
        for file_path in sorted(meta_far_file_paths):
            if file_path == "":
                continue

            content = subprocess.run(
                [
                    ffx_bin,
                    "--isolate-dir",
                    tmpdir,
                    "package",
                    "far",
                    "cat",
                    meta_far_source_path,
                    file_path,
                ],
                stdout=subprocess.PIPE,
            )

            with tempfile.NamedTemporaryFile("wb") as temp_file:
                temp_file.write(content.stdout)
                temp_file.flush()
                file_hash = subprocess.run(
                    [
                        ffx_bin,
                        "--isolate-dir",
                        tmpdir,
                        "package",
                        "file-hash",
                        temp_file.name,
                    ],
                    stdout=subprocess.PIPE,
                    text=True,
                )

                file_content_hash = file_hash.stdout.split()[0]
                meta_far_paths_and_merkles.append(
                    (file_path, file_content_hash)
                )

    return meta_far_paths_and_merkles


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--manifest", help="Path to the package manifest.", required=True
    )
    parser.add_argument(
        "--output",
        help="Path to the content checklist file to compute.",
        required=True,
    )
    parser.add_argument(
        "--ffx-bin", help="Path to ffx tooling.", required=False
    )
    parser.add_argument(
        "--far-bin", help="Path to far tooling.", required=False
    )
    parser.add_argument(
        "--expected-files-exact",
        action="append",
        help="Exact files to capture.",
        required=False,
        default=[],
    )
    parser.add_argument(
        "--expected-files-present",
        action="append",
        help="Present files to capture.",
        required=False,
        default=[],
    )
    parser.add_argument(
        "--reference",
        help="Path to the golden content checklist file",
        required=False,
    )
    parser.add_argument(
        "--warn",
        help="Whether content checklist changes should only cause warnings",
        action="store_true",
    )
    parser.add_argument(
        "--depfile", help="Path for generating depfile.", required=False
    )
    parser.add_argument(
        "--is-coverage",
        help="If yes, hash check will be downgraded to presence check to avoid hash differences from inclusion of debug data (see `//tools/cmc/build/cml.gni` for more).",
        action="store_true",
    )

    args = parser.parse_args()

    if not os.path.isfile(args.manifest):
        print(
            f"Manifest file not found at location '{args.manifest}'. Exiting.",
            file=sys.stderr,
        )
        return 1

    depfile_collection = {args.output: []}
    manifest = {}
    with open(args.manifest, "r") as manifest_file:
        manifest = json.load(manifest_file)

    paths_and_merkles = [
        (blob["path"], blob["merkle"]) for blob in manifest["blobs"]
    ]

    # Retrieve paths and merkles from `meta/` entities.
    if args.ffx_bin and args.far_bin:
        for blob in manifest["blobs"]:
            if blob["path"] == "meta/":
                meta_far_source_path = blob["source_path"]
                meta_far_paths_and_merkles = get_meta_far_contents(
                    args.ffx_bin, args.far_bin, meta_far_source_path
                )
                paths_and_merkles += meta_far_paths_and_merkles
                depfile_collection[args.output] += [meta_far_source_path]

                break

    generated_package_content_checklist = {}
    generated_package_content_checklist["version"] = manifest["version"]
    # OrderedDict used to keep files in sorted order, while keeping top-level
    # values in declared order.
    generated_package_content_checklist["content"] = {
        "files": collections.OrderedDict()
    }
    for path, merkle in paths_and_merkles:
        # Ensure no duplicate files.
        if path in generated_package_content_checklist["content"]["files"]:
            print(
                f"Path found multiple times in manifest: '{path}'",
                file=sys.stderr,
            )
            return 1

        if path in args.expected_files_present:
            # Add as a 'present' file.
            generated_package_content_checklist["content"]["files"][path] = {
                "present": True
            }
        if path in args.expected_files_exact:
            if args.is_coverage:
                # Add as a 'present' file.
                generated_package_content_checklist["content"]["files"][
                    path
                ] = {"present": True}
            else:
                # Add as an 'exact' file.
                generated_package_content_checklist["content"]["files"][
                    path
                ] = {"hash": merkle}

    # Ensure all expected files seen in manifest.
    expected_files_present = args.expected_files_present
    expected_files_exact = args.expected_files_exact
    if args.is_coverage:
        print(
            f"Warning: Hash checks downgraded to presence-checks. See 'is-coverage' flag in `//build/packages/generate_sdk_package_content_checklist.py` for more.",
            file=sys.stdout,
        )
        expected_files_present += args.expected_files_exact
        expected_files_exact = []

    for expected_file_present in expected_files_present:
        if (
            expected_file_present
            not in generated_package_content_checklist["content"][
                "files"
            ].keys()
        ):
            print(
                f"File declared 'present' not found manifest: '{expected_file_present}'. Files available from manifest and meta far are:",
                file=sys.stderr,
            )
            print(
                "\n".join(sorted([path for path, _ in paths_and_merkles])),
                file=sys.stderr,
            )
            return 1
    for expected_file_exact in expected_files_exact:
        if (
            expected_file_exact
            not in generated_package_content_checklist["content"][
                "files"
            ].keys()
        ):
            print(
                f"File declared 'exact' not found manifest: '{expected_file_exact}'. Files available from manifest and meta far are:",
                file=sys.stderr,
            )
            print(
                "\n".join(sorted([path for path, _ in paths_and_merkles])),
                file=sys.stderr,
            )
            return 1

    generated_package_content_checklist_str = json.dumps(
        generated_package_content_checklist, indent=2
    )

    with open(args.output, "w") as output_file:
        output_file.write(generated_package_content_checklist_str)

    # If present, ensure generated file matches golden.
    if args.reference is not None:
        # Absolute path used for more actionable error messages.
        reference_abs_path = os.path.abspath(args.reference)
        depfile_collection[args.output] += [args.reference]
        passed_golden = False
        if not os.path.isfile(args.reference):
            print(
                f"Golden file specified, but no file found at {reference_abs_path}.",
                file=sys.stderr,
            )
        else:
            with open(args.reference, "r") as manifest_file:
                golden = json.load(manifest_file)

            if args.is_coverage:
                # Must change golden to only use presence in case of debugdata.
                for path in golden["content"]["files"]:
                    golden["content"]["files"][path] = {"present": True}
            golden_str = json.dumps(golden, indent=2)

            if not generated_package_content_checklist_str == golden_str:
                print(
                    "Error: SDK package golden and generated content checklist file do not match.",
                    file=sys.stderr,
                )
                print(
                    "\n".join(
                        difflib.unified_diff(
                            golden_str.splitlines(),
                            generated_package_content_checklist_str.splitlines(),
                        )
                    ),
                    file=sys.stderr,
                )
            else:
                passed_golden = True

        if not passed_golden:
            print(
                "To overwrite the golden file location with the newly generated content checklist file, issue this command:",
                file=sys.stderr,
            )
            print(
                f'  mkdir -p "{os.path.dirname(reference_abs_path)}" && cp {os.path.abspath(args.output)} {reference_abs_path}',
                file=sys.stderr,
            )
            if not args.warn:
                return 1

    # Write out depfile
    if args.depfile and len(depfile_collection[args.output]) > 0:
        os.makedirs(os.path.dirname(args.depfile), exist_ok=True)
        with open(args.depfile, "w") as f:
            for out_file in sorted(depfile_collection.keys()):
                in_file_list = sorted(depfile_collection[out_file])
                f.write(f"{out_file}: {' '.join(in_file_list)}")

    return 0


if __name__ == "__main__":
    sys.exit(main())