scripts/codesize/bin/main.dart

// Copyright 2020 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// @dart = 2.8

import 'dart:async';
import 'dart:collection';
import 'dart:convert';
import 'dart:core';
import 'dart:io';

import 'package:crypto/crypto.dart' show sha256;
import 'package:pool/pool.dart';
import 'package:path/path.dart' as path;
import 'package:googleapis/discovery/v1.dart' as discovery;

import 'package:codesize/codesize.dart';
import 'package:codesize/crash_handling.dart' as crash_handling;
import 'package:codesize/cli.dart' as cli;
import 'package:codesize/io.dart' as io;

import 'progress_bar.dart';

/// The common location for storing Fuchsia debug symbols. See fxbug.dev/41031.
const _fxSymbolCache = '.fuchsia/debug/symbol-cache';

/// Entry point of our application from the command line.
Future<void> main(List<String> args) =>
    runWithIo<io.Standard, void>(() => mainImpl(args));

Future<void> mainImpl(List<String> args) async {
  final parsedArgs = cli.parseArgs(args);
  if (parsedArgs == null) return;

  await crash_handling.withExceptionHandler(() async {
    // The high-level flow of the program:
    // - Generate bloaty reports if they aren't already cached.
    // - Run requested queries on those reports.
    // - Present the results in the specified format.
    await preflightChecks();

    final cs = CodeSize(parsedArgs.buildDir);

    AnalysisRequest allBloatyReportFiles = await ensureReportFiles(cs,
        cachingBehavior: parsedArgs.cachingBehavior,
        heatmap: parsedArgs.heatmap,
        heatmapFrameSize: parsedArgs.heatmapFrameSize);

    List<Query> populatedQueries = await runQueriesOnReports(
        parsedArgs.selectedQueries,
        parsedArgs.concurrency,
        parsedArgs.onlyLang,
        parsedArgs.fileRegex,
        allBloatyReportFiles,
        cs);

    await presentResults(
        parsedArgs.format, parsedArgs.output, populatedQueries);
  });
}

/// Checks preconditions for running codesize:
/// - The CPU architecture should be arm64.
Future<void> preflightChecks() async {
  final fxEnv = Io.get().fxEnv;
  if (fxEnv.fuchsiaArch != 'arm64') {
    throw crash_handling.KnownFailure('`fx codesize` is only supported '
        'in arm64 builds. The current architecture is ${fxEnv.fuchsiaArch}.\n'
        'Please switch to the correct product using `fx set`.');
  }
}

class ParseManifestResult {
  /// List of `meta.far` files indicating packages.
  final List<File> packageFars;

  /// A mapping from Merkel roots to blob information.
  final Map<String, Blob> blobsByHash;

  ParseManifestResult(this.packageFars, this.blobsByHash);
}

class CodeSize {
  CodeSize(String buildDir) : build = Build(buildDir);

  final Build build;

  /// List of packages in the build.
  final List<Package> packages = <Package>[];

  /// A mapping from BuildIds to build artifacts with that BuildId.
  /// There might be multiple artifacts with the same BuildId. For instance,
  /// the same binary might appear in both fuchsia.zbi and zedboot.zbi.
  HashMap<String, SplayTreeSet<BuildArtifact>> artifactsByBuildId =
      HashMap<String, SplayTreeSet<BuildArtifact>>();

  /// A mapping from BuildIds to lists of files with that BuildId.
  final HashMap<String, SplayTreeSet<File>> debugBinaries =
      HashMap<String, SplayTreeSet<File>>();

  /// A mapping from BuildIds to link map files.
  final HashMap<String, File> buildIdToLinkMapFile = HashMap<String, File>();

  /// The set of BuilIds for which we have obtained a debug binary.
  final Set<String> matchedDebugBinaries = <String>{};

  // Callbacks used to control the progress bar.
  void Function(int) jobInitCallback;
  void Function() jobIterationCallback;
  void Function() jobCompleteCallback;

  /// I/O operations implementation.
  final Io io = Io.get();

  Future<ParseManifestResult> parseManifest(File manifestFile) async {
    final List<File> packageFars = <File>[];
    final Map<String, Blob> blobsByHash = {};

    final manifestDir = manifestFile.parent;
    final lines = await manifestFile.readAsLines();
    // Format of the manifest:
    // HASH = RELATIVE PATH
    for (final line in lines) {
      final parts = line.split('=');
      final hash = parts[0];
      // Path entries are specified relative to the directory
      // containing the manifest.
      final entryPath = build.rebasePath((manifestDir / File(parts[1])).path);
      final file = build.openFile(entryPath);
      if (entryPath.endsWith('meta.far')) {
        // Blobs ending with `meta.far` indicate there is a
        // corresponding package.
        packageFars.add(file);
      }

      final stat = file.statSync();
      if (stat.type == FileSystemEntityType.notFound) {
        throw Exception('$entryPath does not exist');
      }
      if (blobsByHash.containsKey(hash)) {
        // Duplicate blob entry.
        io.err.writeln(
            'Duplicate blob entry: $hash -> ${build.rebasePath(entryPath)} '
            'and ${blobsByHash[hash].buildPath}');
        continue;
      }
      blobsByHash[hash] = await createBlob(hash, entryPath, stat);
    }
    return ParseManifestResult(
        packageFars.toList(growable: false), blobsByHash);
  }

  Future<Blob> createBlob(String hash, String entryPath, FileStat stat) async {
    final blob = Blob()
      ..hash = hash
      ..buildPath = build.rebasePath(entryPath)
      ..sizeOnHost = stat.size
      ..count = 0;

    // Extract contents as subBlobs if the blob is a Zircon Boot Image.
    if (entryPath.endsWith('.zbi.signed')) {
      blob.subBlobs = await extractZbi(entryPath);
    }
    return blob;
  }

  /// Extracts the ZBI inside the build directory and returns the contents
  /// as a list of `SubBlob`s.
  Future<List<SubBlob>> extractZbi(String entryPath) async {
    // Signed ZBI cannot be processed directly by the zbi tool.
    // Use the unsigned counterpart.
    final unsignedZbi =
        build.openFile(entryPath.substring(0, entryPath.length - 7));
    final bootfsDir = build.openDirectory(
        'obj/codesize/bootfs-${unsignedZbi.path.split('/').last}');
    if (bootfsDir.existsSync()) {
      // Clear previously extracted directory.
      await bootfsDir.delete(recursive: true);
    }
    await bootfsDir.create(recursive: true);
    print('Extracting ${build.rebasePath(unsignedZbi.path)}');
    // Run the zbi host tool.
    // out/default/host_x64/zbi --extract --output-dir ~/vg/out/tmp out/default/fuchsia.zbi
    final result = await io.processManager.run([
      build.openFile('host_x64/zbi').path,
      '--extract',
      '--output-dir',
      bootfsDir.absolute.path,
      unsignedZbi.absolute.path
    ],
        workingDirectory: build.dir.path,
        stdoutEncoding: const AsciiCodec(allowInvalid: true),
        stderrEncoding: const AsciiCodec(allowInvalid: true));
    if (result.exitCode != 0) {
      print('stdout: ${result.stdout}');
      print('stderr: ${result.stderr}');
      throw Exception('Failed to extract zbi for $unsignedZbi');
    }
    final files = bootfsDir.list(recursive: true, followLinks: false);
    final subBlobs = <SubBlob>[];
    await for (final file in files) {
      if (file is File) {
        final subBlob = SubBlob()
          ..name =
              path.relative(path.normalize(file.path), from: bootfsDir.path)
          ..buildPath = build.rebasePath(file.path)
          ..sizeOnHost = (file.statSync()).size;
        subBlobs.add(subBlob);
      }
    }
    return subBlobs;
  }

  Future<void> addBlobSizes(File path, Map<String, Blob> blobsByHash) async {
    final blobs = json.decode(await path.readAsString());
    for (var blob in blobs) {
      final b = blobsByHash[blob['merkle']];
      b?.size = blob['size'];
    }
  }

  String metaFarPathToBlobsJson(File far) {
    // Assumes details of //build/package.gni, namely that it generates
    //   <build-dir>/.../<package>/meta.far
    // and puts a blobs.json file into
    //   <build-dir>/.../<package>/blobs.json
    if (!far.path.endsWith('/meta.far')) {
      throw Exception('Build details have changed');
    }
    final jsonPath = "${removeSuffix(far.path, 'meta.far')}blobs.json";
    if (!build.openFile(jsonPath).existsSync()) {
      throw Exception('Build details have changed - '
          'path to blobs.json $jsonPath not found for ${far.path}');
    }
    return jsonPath;
  }

  Future<void> computePackagesInParallel(
      ParseManifestResult manifest, int jobs) async {
    final tasks = <Future>[];
    for (var i = 0; i < jobs; i++) {
      tasks.add(computePackages(manifest));
    }
    await Future.wait(tasks);
  }

  Future<void> computePackages(ParseManifestResult manifest) async {
    final packageFars = manifest.packageFars.toList();
    while (packageFars.isNotEmpty) {
      File far = packageFars.removeLast();

      var package = Package()..path = far.path;
      var parts = package.path.split('/');
      package
        ..name = maybeRemoveSuffix(
            parts.length > 1 ? parts[parts.length - 2] : parts.last, '.meta')
        ..size = 0
        ..private = 0
        ..blobCount = 0
        ..blobsByPath = <String, Blob>{};

      var blobs = json.decode(
          await build.openFile(metaFarPathToBlobsJson(far)).readAsString());

      for (var blob in blobs) {
        var hash = blob['merkle'];
        var blobPath = build.rebasePath(blob['path']);
        var b = manifest.blobsByHash[hash];
        if (b == null) {
          throw Exception('$blobPath $hash is in a package manifest '
              'but not the final manifest');
        }
        b.count++;
        var sourcePath = build.openFile(blob['source_path']).path;
        // If the source_path looks like <something>/blobs/<merkle>, it from a
        // prebuilt package and has no meaningful source. Instead, use the path
        // within the package as its identifier.
        if (sourcePath.endsWith('/blobs/$hash')) {
          sourcePath = 'pkg:${package.name}/${build.rebasePath(blobPath)}';
        } else {
          sourcePath = build.rebasePath(sourcePath);
        }
        // We may see the same blob referenced from different packages with
        // different source paths. If all references agree with each other, use
        // that. Otherwise we would print the first observed path and
        // append " *" to denote that the path is only one of many.
        b.sourcePaths.add(sourcePath);
        package.blobsByPath[blobPath] = b;
      }

      packages.add(package);
    }
  }

  void computeStats(Map<String, Blob> blobsByHash) {
    blobsByHash.forEach((hash, blob) {
      if (blob.count == 0) {
        throw Exception('${blob.hash} is in the final manifest '
            'but not any package manifest');
      }
    });

    for (var package in packages) {
      for (var blob in package.blobsByPath.values) {
        package.size += blob.size;
        if (blob.count == 1) {
          package.private += blob.size;
        }
        package.blobCount++;
      }
    }
  }

  Future<void> addBuildIdFromBlobsInImage(Map<String, Blob> blobsByHash) async {
    final artifacts = blobsByHash.values.expand((blob) {
      return <BuildArtifact>[blob] + blob.subBlobs;
    });
    artifactsByBuildId = await readBuildId(
        artifacts,
        (BuildArtifact art) => build.openFile(art.buildPath),
        Platform.numberOfProcessors * 2);
  }

  Future<void> addBuildIdFromSymbolSources() async {
    final fuchsiaBuildDir = build.openDirectory('.build-id');
    final Directory zirconBuildDir =
        build.zirconBuildDirectory() / Directory('.build-id');
    final sourceTree = Directory(Platform.environment['FUCHSIA_DIR']).absolute;
    final home = Directory(Platform.environment['HOME']);
    final maybeCipdBuildIdDirs = ([
              'prebuilt/.build-id',
              'prebuilt/third_party/clang/linux-x64/lib/debug/.build-id',
              'prebuilt/third_party/clang/mac-x64/lib/debug/.build-id',
            ].map((e) => sourceTree / Directory(e)).toList() +
            [
              home / Directory(_fxSymbolCache),
            ])
        .where((e) => e.existsSync());
    await _addBuildId(Stream.fromIterable(
        [fuchsiaBuildDir, zirconBuildDir, ...maybeCipdBuildIdDirs]));
  }

  Future<void> _addBuildId(Stream<Directory> dirs) async {
    final debugFiles = <File>[];
    final contents = dirs.asyncExpand((dir) {
      if (!dir.existsSync()) {
        throw Exception('Cannot find ${dir.absolute}');
      }
      return dir.list(recursive: true, followLinks: false).where(
          (entity) => (entity is File) && entity.path.endsWith('.debug'));
    });
    await for (final entity in contents) {
      if (entity is File) {
        debugFiles.add(entity);
      }
    }
    debugBinaries.addAll(await readBuildId(
        debugFiles, (x) => x, Platform.numberOfProcessors * 2,
        compare: (File a, File b) => a.path.compareTo(b.path)));
  }

  void matchDebugBinaries() {
    for (final buildId in debugBinaries.keys) {
      if (artifactsByBuildId.containsKey(buildId)) {
        matchedDebugBinaries.add(buildId);
      }
    }
  }

  Future<HashMap<String, SplayTreeSet<T>>> readBuildId<T>(
      Iterable<T> items, File Function(T) toFile, int jobs,
      {int compare(T a, T b)}) async {
    jobInitCallback?.call(items.length);
    final exp = RegExp(
        r'(.*): ELF 64-bit LSB .*ARM aarch64,.*BuildID(.*)=([a-f0-9]+),');
    final buildIdToBlob = HashMap<String, SplayTreeSet<T>>();
    final pool = Pool(jobs);
    final allFutures = <Future>[];
    for (final blob in items) {
      final file = toFile(blob);
      allFutures.add(pool.withResource(() async {
        final result =
            await io.processManager.run(['file', file.absolute.path]);
        if (result.exitCode != 0) {
          print(result.stdout);
          print(result.stderr);
          throw Exception('Failed to inspect $file');
        }
        final matches = exp.allMatches(result.stdout);
        if (matches.isNotEmpty) {
          if (matches.length != 1) {
            throw Exception('Unexpected matches on `file $file`');
          }
          final match = matches.first;
          // Fuchsia uses [xxHash]; Chromium uses [sha1]
          final buildIdAlgo = match.group(2);
          if (buildIdAlgo != '[xxHash]' &&
              buildIdAlgo != '[sha1]' &&
              buildIdAlgo != '[md5/uuid]') {
            throw Exception(
                'Unexpected BuildId algo `$buildIdAlgo` on `file $file`');
          }
          final buildId = match.group(3);
          buildIdToBlob.putIfAbsent(buildId, () => SplayTreeSet<T>(compare));
          buildIdToBlob[buildId].add(blob);
        }
        jobIterationCallback?.call();
      }));
    }
    await Future.wait(allFutures);
    jobCompleteCallback?.call();
    return buildIdToBlob;
  }

  // Finds all `.map` files in the build directory, and find the corresponding
  // BuildID of the unstripped binary that is next to it.
  Future<void> collectLinkMapFiles() async {
    final files = build.dir.list(recursive: true, followLinks: false).where(
        (file) =>
            file.path.endsWith('.map') &&
            file.statSync().type == FileSystemEntityType.file);
    final elfFiles = <String>[];
    await for (var mapFile in files) {
      final elfFile =
          build.openFile(mapFile.path.substring(0, mapFile.path.length - 4));
      if (!elfFile.existsSync()) continue;
      elfFiles.add(elfFile.path);
    }
    final buildIds = await readBuildId(
        elfFiles, build.openFile, Platform.numberOfProcessors * 2);
    for (final entry in buildIds.entries) {
      buildIdToLinkMapFile[entry.key] =
          build.openFile('${entry.value.first}.map');
    }
  }
}

Future<AnalysisRequest> ensureReportFiles(CodeSize cs,
    {cli.CachingBehavior cachingBehavior,
    File heatmap,
    int heatmapFrameSize}) async {
  AnalysisRequest allBloatyReportFiles;
  final bloatyStamp = cs.build.openFile('codesize_bloaty_report.stamp');
  if (bloatyStamp.existsSync()) {
    bool useCache;
    switch (cachingBehavior) {
      case cli.CachingBehavior.alwaysUseCache:
        print('Explicitly using cached report files');
        useCache = true;
        break;
      case cli.CachingBehavior.neverUseCache:
        print('Ignoring cached report files');
        useCache = false;
        break;
      case cli.CachingBehavior.useIfUpToDate:
        // Use the cached reports if no newer full build was performed.
        final manifestStat = cs.build.blobManifestFile().statSync();
        final stampStat = bloatyStamp.statSync();
        useCache = stampStat.modified.isAfter(manifestStat.modified);
        if (useCache) {
          // Check if heatmap file remained the same too.
          final lastRequest = AnalysisRequest.fromJson(
              json.decode(await bloatyStamp.readAsString()));
          if (lastRequest.heatmapContentSha != null) {
            if (heatmap == null) {
              useCache = false;
            } else {
              final digest = await _hashFile(heatmap);
              if (digest != lastRequest.heatmapContentSha) {
                useCache = false;
              }
            }
          } else {
            if (heatmap != null) {
              useCache = false;
            }
          }
          if (!useCache) {
            print('Not using cached report files since '
                'the heatmap option has changed');
          }
        } else {
          print('Not using cached report files since '
              'a new full build has been performed');
        }
        if (useCache) {
          print('Using cached report files since '
              'it was generated after the last full build');
        }
        break;
    }
    if (useCache) {
      // Skipping running bloaty; load `allBloatyReportFiles` from stamp
      allBloatyReportFiles = AnalysisRequest.fromJson(
          json.decode(await bloatyStamp.readAsString()));
      print('Loaded ${allBloatyReportFiles.items.length} reports from cache');
      return allBloatyReportFiles;
    }
  }

  // Rerun bloaty and save the report file index as a stamp.
  try {
    allBloatyReportFiles = await generateBloatyReportsFromBuild(cs,
        heatmap: heatmap, heatmapFrameSize: heatmapFrameSize);
    await bloatyStamp.writeAsString(json.encode(allBloatyReportFiles.toJson()));
  } finally {
    await GoogleApiClient.close();
  }
  return allBloatyReportFiles;
}

Future<String> _hashFile(File heatmap) async =>
    sha256.convert(await heatmap.readAsBytes()).toString();

Future<List<Query>> runQueriesOnReports(
    List<QueryThunk> queries,
    int concurrency,
    SourceLang onlyLang,
    RegExp fileRegex,
    AnalysisRequest allBloatyReportFiles,
    CodeSize cs) async {
  print('Running these queries: ${queries.map((s) => s().name).join(', ')}');
  final queryRunner =
      QueryRunner(queries, numConcurrency: concurrency, onlyLang: onlyLang);
  final filteredReports = allBloatyReportFiles.items
      .where((report) => fileRegex.hasMatch(report.name));
  print(
      'Analyzing ${filteredReports.length} reports whose names match the ${fileRegex.pattern} regex');
  await Future.wait([
    for (final reportFile in filteredReports)
      queryRunner.addReport(reportFile.absolute(cs.build.dir.path))
  ]);
  await queryRunner.join();
  return queryRunner.queries;
}

Future<void> presentResults(cli.OutputFormat outputFormat, IOSink output,
    List<Query> populatedQueries) async {
  final io = Io.get();
  // Data presentation
  Renderer renderer;
  switch (outputFormat) {
    case cli.OutputFormat.basic:
      renderer = BasicRenderer();
      break;
    case cli.OutputFormat.html:
      renderer = HtmlRenderer();
      break;
    case cli.OutputFormat.terminal:
      renderer = TerminalRenderer(supportsControlCharacters: output == io.out);
      break;
    case cli.OutputFormat.tsv:
      renderer = TsvRenderer();
      break;
  }

  try {
    renderer.render(output, populatedQueries);
  } finally {
    await output.flush();
    if (output != io.out) {
      await output.close();
    }
  }
}

Future<AnalysisRequest> generateBloatyReportsFromBuild(CodeSize cs,
    {File heatmap, int heatmapFrameSize}) async {
  final allBloatyReportFiles = AnalysisRequest(
      items: [], heatmapContentSha: await flatMap(heatmap, _hashFile));
  final io = Io.get();

  final manifest = await cs.parseManifest(cs.build.blobManifestFile());
  final blobsByHash = manifest.blobsByHash;
  await cs.addBlobSizes(cs.build.openFile('blobs.json'), blobsByHash);
  await cs.computePackagesInParallel(manifest, Platform.numberOfProcessors);
  cs.computeStats(blobsByHash);

  ProgressBar progress;
  cs
    ..jobInitCallback = ((max) => progress = ProgressBar(complete: max))
    ..jobIterationCallback = (() => progress.update(progress.current + 1))
    ..jobCompleteCallback = (() => progress.done());

  io.out.write('Loading link maps ');
  await cs.collectLinkMapFiles();

  io.out.write('Loading BuildId for blobs in image ');
  await cs.addBuildIdFromBlobsInImage(blobsByHash);
  final elfBlobSizes = cs.artifactsByBuildId.values
      .map((e) => e.first.sizeOnHost)
      .reduce((v, e) => v + e);
  final allBlobSizes =
      blobsByHash.values.map((e) => e.sizeOnHost).reduce((v, e) => v + e);
  print('${cs.artifactsByBuildId.length} (${formatSize(elfBlobSizes)}) out of '
      '${blobsByHash.length} (${formatSize(allBlobSizes)}) blobs are '
      'ELF binaries');

  io.out.write('Loading BuildId from symbol sources ');
  await cs.addBuildIdFromSymbolSources();
  print('${cs.debugBinaries.length} debug binaries found locally');

  cs.matchDebugBinaries();
  final matchedBinarySizes = cs.matchedDebugBinaries
      .map((k) => cs.artifactsByBuildId[k].first.sizeOnHost)
      .reduce((v, e) => v + e);
  print('${cs.matchedDebugBinaries.length} (${formatSize(matchedBinarySizes)})'
      ' out of ${cs.artifactsByBuildId.length} (${formatSize(elfBlobSizes)})'
      ' ELF binaries have matching debug info');

  Iterable<String> buildIdsWithoutDebugInfo() => cs.artifactsByBuildId.keys
      .where((buildId) => !cs.matchedDebugBinaries.contains(buildId));

  {
    final noDebugInfo = buildIdsWithoutDebugInfo();
    if (noDebugInfo.isNotEmpty) {
      io.out.write('Querying symbol servers ');
      final gotAny = await downloadUnmatchedDebugBinaries(noDebugInfo, cs);
      if (gotAny) {
        io.out.write('Incorporate downloaded symbols ');
        await cs.addBuildIdFromSymbolSources();
        cs.matchDebugBinaries();
      } else {
        print('Did not find any extra symbols from symbol servers');
      }
    }
  }

  {
    // Print binaries without debug info even after querying symbol servers.
    final noDebugInfo = buildIdsWithoutDebugInfo();
    if (noDebugInfo.isNotEmpty) {
      print('Binaries without matching debug info:');
      for (final buildId in noDebugInfo) {
        final buildPath = cs.artifactsByBuildId[buildId].first.buildPath;
        print('  ${cs.build.rebasePath(buildPath)}');
        print('  - BuildId: $buildId');
        if (blobsByHash.containsKey(buildPath.split('/').last)) {
          final sourcePath = blobsByHash[buildPath.split('/').last].sourcePaths;
          print('  - Source Path: $sourcePath');
        }
      }
    }
  }

  var buildIds = <String>{};
  HashMap<String, String> buildIdToAccessPattern;
  if (heatmap != null) {
    buildIdToAccessPattern = HashMap<String, String>();
    final HashMap<String, String> merkleToAccessPattern =
        HashMap<String, String>();
    for (final line in await heatmap.readAsLines()) {
      if (line.trim().isEmpty) continue;
      final firstComma = line.indexOf(',');
      final merkle = line.substring(0, firstComma);
      final accessPattern = line.substring(firstComma + 1);
      merkleToAccessPattern[merkle] = accessPattern;
    }
    for (final entry in cs.artifactsByBuildId.entries) {
      final buildId = entry.key;
      final artifact = entry.value.first;
      if (artifact is Blob) {
        if (cs.matchedDebugBinaries.contains(buildId) &&
            merkleToAccessPattern.containsKey(artifact.hash)) {
          final accessPattern = merkleToAccessPattern[artifact.hash];
          // Exclude fully-hot blobs, since downstream queries would be confused
          // as to which language this ELF is written in due to lack of symbols.
          final int elfSize =
              cs.build.openFile(artifact.buildPath).statSync().size;
          final int numFrames = (elfSize / heatmapFrameSize).ceil();
          final isFrameHot = List<bool>.generate(numFrames, (i) => false);
          for (final part in accessPattern.split(',')) {
            final frameAndCount = part.split(':').toList();
            if (frameAndCount.length != 2) {
              throw Exception('Invalid access pattern $accessPattern');
            }
            final frame = int.parse(frameAndCount[0]);
            final count = int.parse(frameAndCount[1]);
            if (frame >= numFrames) {
              throw Exception('Blob merkle ${artifact.hash} '
                  '(at: ${artifact.buildPath}) is $elfSize bytes on disk, '
                  'but access pattern indicated an out-of-range frame $frame. '
                  'Does the heatmap CSV match the version of the build?');
            }
            if (count > 0) isFrameHot[frame] = true;
          }
          if (isFrameHot.every((e) => e)) continue;

          buildIds.add(buildId);
          buildIdToAccessPattern[buildId] = accessPattern;
        }
      }
    }

    io.out.writeln('Blob access heatmap specified, only looking at '
        '${buildIds.length} files with both cold regions and debug info');
    for (final buildId in buildIds) {
      io.out.writeln('-> ${cs.artifactsByBuildId[buildId].first}');
    }
  } else {
    buildIds = cs.matchedDebugBinaries;
  }

  io.out.write('Running bloaty on matched binaries ');
  await runBloatyOnMatchedBinaries(buildIds,
      options: RunBloatyOptions(
        build: cs.build,
        artifactsByBuildId: cs.artifactsByBuildId,
        debugBinaries: cs.debugBinaries,
        buildIdToLinkMapFile: cs.buildIdToLinkMapFile,
        buildIdToAccessPattern: buildIdToAccessPattern,
        heatmapFrameSize: heatmapFrameSize,
        jobInitCallback: cs.jobInitCallback,
        jobIterationCallback: cs.jobIterationCallback,
        jobCompleteCallback: cs.jobCompleteCallback,
      ));

  final zbiExtractedBinary = RegExp(r'obj/codesize/bootfs-(.*)\.zbi/(.*)$');
  for (final buildId in buildIds) {
    final blob = cs.artifactsByBuildId[buildId].first;
    var name = blob.buildPath;
    if (blob is SubBlob) {
      final match = zbiExtractedBinary.firstMatch(blob.buildPath);
      if (match != null)
        name = '[zbi: ${match.group(1)}.zbi] /${match.group(2)} '
            '(at: ${cs.build.rebasePath(blob.buildPath)})';
    } else if (blob is Blob) {
      final formattedBuildPath = cs.build.rebasePath(blob.buildPath);
      final prebuiltDueToPkgHash = blob.sourcePaths
          .fold(true, (prev, element) => prev && element.startsWith('pkg:'));
      final prebuiltInPrebuiltFolder =
          formattedBuildPath.startsWith('../../prebuilt/');
      if (prebuiltDueToPkgHash || prebuiltInPrebuiltFolder) {
        name = '[prebuilt] '
            '${blob.sourcePaths.first}'
            '${blob.sourcePaths.length > 1 ? "* " : ""} '
            '(at: $formattedBuildPath)';
      } else {
        final maybeNonPkgPrebuilt =
            blob.sourcePaths.where((e) => !e.startsWith('pkg:')).first;
        if (maybeNonPkgPrebuilt != null) {
          name = maybeNonPkgPrebuilt;
        } else {
          name = formattedBuildPath;
        }
      }
    }
    // If heatmap was specified, we'd generate two versions of reports, one of
    // which is filtered using the heatmap.
    final report = AnalysisItem(
        path: cs.build.rebasePath('${blob.buildPath}.bloaty_report_pb'),
        filteredCounterpart: heatmap != null
            ? cs.build.rebasePath('${blob.buildPath}.filtered.bloaty_report_pb')
            : null,
        name: name);
    if (!cs.build.openFile(report.path).existsSync())
      throw Exception('Could not find bloaty report at ${report.path}');
    allBloatyReportFiles.items.add(report);
  }

  return allBloatyReportFiles;
}

Future<bool> downloadUnmatchedDebugBinaries(
    Iterable<String> buildIds, CodeSize cs) async {
  final homeDir = Directory(Platform.environment['HOME']);
  final repos = await Future.wait([
    'gs://fuchsia-artifacts/debug',
    'gs://fuchsia-artifacts-internal/debug',
    'gs://fuchsia-artifacts-release/debug',
  ].map(
      (e) => CloudRepo.create(e, Cache(homeDir / Directory(_fxSymbolCache)))));
  var downloadedAny = false;
  cs.jobInitCallback?.call(buildIds.length);

  // 2 concurrent symbol downloads
  final pool = Pool(2);
  final allFutures = <Future>[];
  for (final buildId in buildIds) {
    allFutures.add(pool.withResource(() async {
      var downloaded = false;
      for (var repo in repos) {
        try {
          await repo.getBuildObject(buildId);
          // A 404 error will be expected if the debug symbol is not found.
          // ignore: avoid_catching_errors
        } on discovery.DetailedApiRequestError catch (err) {
          if (err.status != 404) {
            rethrow;
          }
          continue;
        }
        downloaded = true;
        break;
      }
      if (downloaded) downloadedAny = true;
      cs.jobIterationCallback?.call();
    }));
  }
  await Future.wait(allFutures);
  cs.jobCompleteCallback?.call();
  return downloadedAny;
}