cmd/dump_breakpad_symbols/main.go

// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	// TODO(kjharland): change crypto/sha1 to a safer hash algorithm. sha256 or sha2, etc.
	"archive/tar"
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"fuchsia.googlesource.com/tools/breakpad"
	"fuchsia.googlesource.com/tools/elflib"
)

const usage = `usage: dump_breakpad_symbols [options] file1 file2 ... fileN

Dumps symbol data from a collection of IDs files. IDs files are generated as
part of the build and contain a number of newline-separate records which have
the syntax:

  <hash-value> <absolute-path>

This command does not care about <hash-value>.  <absolute-path> is the path to a
binary generated as part of the Fuchsia build. This command collects every
<absolute-path> from each of file1, file2 ... fileN and dumps symbol data for
the binaries at each of those paths.  Duplicate paths are skipped.

The output is a collection of symbol files, one for each binary, using an
arbitrary naming scheme to ensure that every output file name is unique.

Example invocation:

$ dump_breakpad_symbols \
	-out-dir=/path/to/output/ \
	-dump-syms-path=/path/to/breakpad/dump_syms \
	-summary-file=/path/to/summary \
	/path/to/ids1.txt
`

// The default module name for modules that don't have a soname, e.g., executables and
// loadable modules. This allows us to use the same module name at runtime as sonames are
// the only names that are guaranteed to be available at build and run times. This value
// must be kept in sync with what Crashpad uses at run time for symbol resolution to work
// properly.
const defaultModuleName = "<_>"

// Command line flag values
var (
	depFilepath  string
	dumpSymsPath string
	outdir       string
	tarFilepath  string
)

// ExecDumpSyms runs the beakpad `dump_syms` command and returns the output.
type ExecDumpSyms = func(args []string) ([]byte, error)

// CreateFile returns an io.ReadWriteCloser for the file at the given path.
type CreateFile = func(path string) (io.ReadWriteCloser, error)

func init() {
	flag.Usage = func() {
		fmt.Fprint(os.Stderr, usage)
		flag.PrintDefaults()
		os.Exit(0)
	}

	// First set the flags ...
	flag.StringVar(&outdir, "out-dir", "",
		"The directory where symbol output should be written")
	flag.StringVar(&dumpSymsPath, "dump-syms-path", "",
		"Path to the breakpad tools `dump_syms` executable")
	flag.StringVar(&depFilepath, "depfile", "",
		"Path to the ninja depfile to generate.  The file has the single line: "+
			"`OUTPUT: INPUT1 INPUT2 ...` where OUTPUT is the value of -summary-file "+
			"and INPUTX is the ids file in the same order it was provided on the "+
			"command line. -summary-file must be provided with this flag. "+
			"See `gn help depfile` for more information on depfiles.")
	flag.StringVar(&tarFilepath, "tar-file", "",
		"Path to the tarball that contains all Breakpad symbol files generated "+
			" by the build")
}

func main() {
	flag.Parse()
	if err := execute(context.Background()); err != nil {
		log.Fatal(err)
	}
}

func execute(ctx context.Context) error {
	// Callback to run breakpad `dump_syms` command.
	execDumpSyms := func(args []string) ([]byte, error) {
		return exec.Command(dumpSymsPath, args...).Output()
	}

	// Callback to create new files.
	createFile := func(path string) (io.ReadWriteCloser, error) {
		return os.Create(path)
	}
	// Open the input files for reading.  In practice there are very few files,
	// so it's fine to open them all at once.
	var inputReaders []io.Reader
	inputPaths := flag.Args()
	for _, path := range inputPaths {
		file, err := os.Open(path)
		if err != nil {
			return fmt.Errorf("failed to open %s: %v\n", path, err)
		}
		defer file.Close()
		inputReaders = append(inputReaders, file)
	}

	// Process the IDsFiles.
	summary := processIdsFiles(inputReaders, outdir, execDumpSyms, createFile)

	// Write the Ninja dep file.
	depfile := depfile{outputPath: tarFilepath, inputPaths: inputPaths}
	depfd, err := os.Create(depFilepath)
	if err != nil {
		return fmt.Errorf("failed to create file %q: %v", depFilepath, err)
	}
	n, err := depfile.WriteTo(depfd)
	if err != nil {
		return fmt.Errorf("failed to write Ninja dep file %q: %v", depFilepath, err)
	}
	if n == 0 {
		return fmt.Errorf("wrote 0 bytes to %q", depFilepath)
	}

	// Create and open tarball.
	tarFile, err := os.Create(tarFilepath)
	if err != nil {
		return fmt.Errorf("could not create file %s: %v", tarFilepath, err)
	}
	defer tarFile.Close()

	// Collect Breakpad symbol files generated by the build to a tarball.
	if err := writeTarball(tarFile, summary, outdir); err != nil {
		return fmt.Errorf("failed to generate tarball %s: %v", tarFilepath, err)
	}
	return nil
}

// processIdsFiles dumps symbol data for each executable in a set of ids files.
func processIdsFiles(idsFiles []io.Reader, outdir string, execDumpSyms ExecDumpSyms, createFile CreateFile) map[string]string {
	// Binary paths we've already seen.  Duplicates are skipped.
	visited := make(map[string]bool)
	binaryToSymbolFile := make(map[string]string)

	// Iterate through the given set of filepaths.
	for _, idsFile := range idsFiles {
		// Extract the paths to each binary from the IDs file.
		binaries, err := elflib.ReadIDsFile(idsFile)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		// Generate the symbol file for each binary.
		for _, bin := range binaries {
			binaryPath := bin.Filepath

			// Check whether we've seen this path already. Skip if so.
			if _, ok := visited[binaryPath]; ok {
				continue
			}
			// Record that we've seen this binary path.
			visited[binaryPath] = true

			symbolFilepath, err := generateSymbolFile(binaryPath, createFile, execDumpSyms)
			if err != nil {
				log.Println(err)
				continue
			}

			// Record the mapping in the summary.
			binaryToSymbolFile[binaryPath] = symbolFilepath
		}
	}

	return binaryToSymbolFile
}

func generateSymbolFile(path string, createFile CreateFile, execDumpSyms ExecDumpSyms) (outputPath string, err error) {
	outputPath = createSymbolFilepath(outdir, path)
	output, err := execDumpSyms([]string{path})
	if err != nil {
		return "", fmt.Errorf("failed to generate symbol data for %s: %v", path, err)
	}
	symbolFile, err := breakpad.ParseSymbolFile(bytes.NewReader(output))
	if err != nil {
		return "", fmt.Errorf("failed to read dump_syms output: %v", err)
	}
	// Ensure the module name is either the soname (for shared libraries) or the default
	// value (for executables and loadable modules).
	fd, err := os.Open(path)
	if err != nil {
		return "", fmt.Errorf("failed to open %q: %v", path, err)
	}
	defer fd.Close()
	soname, err := elflib.GetSoName(path, fd)
	if err != nil {
		return "", fmt.Errorf("failed to read soname from %q: %v", path, err)
	}
	if soname == "" {
		symbolFile.ModuleSection.ModuleName = defaultModuleName
	} else {
		symbolFile.ModuleSection.ModuleName = soname
	}

	// Ensure the module section specifies this is a Fuchsia binary instead of Linux
	// binary, which is the default for the dump_syms tool.
	symbolFile.ModuleSection.OS = "Fuchsia"
	symbolFd, err := createFile(outputPath)
	if err != nil {
		return "", fmt.Errorf("failed to create symbol file %s: %v", outputPath, err)
	}
	defer fd.Close()
	if _, err := symbolFile.WriteTo(symbolFd); err != nil {
		return "", fmt.Errorf("failed to write symbol file %s: %v", outputPath, err)
	}
	return outputPath, nil
}

// Writes the given symbol file data to the given writer after massaging the data.
func writeSymbolFile(w io.Writer, symbolData []byte) error {
	// Many Fuchsia binaries are built as "something.elf", but then packaged as
	// just "something". In the ids.txt file, the name still includes the ".elf"
	// extension, which dump_syms emits into the .sym file, and the crash server
	// uses as part of the lookup.  The binary name and this value written to
	// the .sym file must match, so if the first header line ends in ".elf"
	// strip it off.  This line usually looks something like:
	// MODULE Linux x86_64 094B63014248508BA0636AD3AC3E81D10 sysconf.elf
	lines := strings.SplitN(string(symbolData), "\n", 2)
	if len(lines) != 2 {
		return fmt.Errorf("got <2 lines in symbol data")
	}

	// Make sure the first line is not empty.
	lines[0] = strings.TrimSpace(lines[0])
	if lines[0] == "" {
		return fmt.Errorf("unexpected blank first line in symbol data")
	}

	// Strip .elf from header if it exists.
	if strings.HasSuffix(lines[0], ".elf") {
		lines[0] = strings.TrimSuffix(lines[0], ".elf")
		// Join the new lines of the symbol data.
		symbolData = []byte(strings.Join(lines, "\n"))
	}

	// Write the symbol file.
	_, err := w.Write(symbolData)
	return err
}

// Creates the absolute path to the symbol file for the given binary.
//
// The returned path is generated as a subpath of parentDir.
func createSymbolFilepath(parentDir string, binaryPath string) string {
	// Create the symbole file basename as a hash of the path to the binary.
	// This ensures that filenames are unique within the output directory.
	hash := sha1.New()
	n, err := hash.Write([]byte(binaryPath))
	if err != nil {
		panic(err)
	}
	if n == 0 {
		// Empty text should never be passed to this function and likely signifies
		// an error in the input file. Panic here as well.
		panic("0 bytes written for hash of input text '" + binaryPath + "'")
	}
	basename := hex.EncodeToString(hash.Sum(nil)) + ".sym"

	// Generate the filepath as an subdirectory of the given parent directory.
	absPath, err := filepath.Abs(path.Join(parentDir, basename))
	if err != nil {
		// Panic because if this fails once it's likely to keep failing.
		panic(fmt.Sprintf("failed to get path to symbol file for %s: %v", binaryPath, err))
	}

	return absPath
}

func writeTarball(w io.Writer, summary map[string]string, parentDir string) error {
	gw := gzip.NewWriter(w)
	defer gw.Close()
	// Create a tarWriter to perform the tarring task.
	tw := tar.NewWriter(gw)
	defer tw.Close()

	// Iterate through the summary file, writting each symbol file to the tarball buffer.
	for _, file := range summary {
		data, err := ioutil.ReadFile(file)
		if err != nil {
			return fmt.Errorf("failed to read the symbol data file %s: %v", file, err)
		}

		// Add a file with its relative path instead of absolute path to the tarball.
		rel, err := filepath.Rel(parentDir, file)
		if err != nil {
			return fmt.Errorf("failed to get the relative path from %s to %s: %v", parentDir, file, err)
		}
		hdr := &tar.Header{
			Name: rel,
			Mode: 0600,
			Size: int64(len(data)),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return fmt.Errorf("failed to write header for tarball: %v", err)
		}
		if _, err := tw.Write([]byte(data)); err != nil {
			return fmt.Errorf("failed to write contents for tarball: %v", err)
		}
	}

	return nil
}