tools/debroot/cmd/main.go

// Copyright 2017 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Generates a Debian based sysroot.
package main

import (
	"bufio"
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/google/subcommands"
	"github.com/ulikunitz/xz"
	"golang.org/x/crypto/openpgp"
	"gopkg.in/yaml.v2"
)

const (
	packagesFile = "Packages.xz"
)

type stringsValue []string

func (i *stringsValue) String() string {
	return strings.Join(*i, ",")
}

func (i *stringsValue) Set(value string) error {
	*i = strings.Split(value, ",")
	return nil
}

type Config struct {
	Dists         []string  `yaml:"dists"`
	Components    []string  `yaml:"components"`
	Sources       []string  `yaml:"sources"`
	Keyring       string    `yaml:"keyring"`
	Architectures []string  `yaml:"architectures"`
	Packages      []Package `yaml:"packages"`
}

type Package struct {
	Name          string   `yaml:"package"`
	Architectures []string `yaml:"architectures,omitempty"`
}

type Lockfile struct {
	Hash     string    `yaml:"hash"`
	Updated  time.Time `yaml:"updated"`
	Packages []Lock    `yaml:"packages"`
}

type Lock struct {
	Name    string `yaml:"package"`
	Version string `yaml:"version"`
	Url     string `yaml:"url"`
	Hash    string `yaml:"hash"`
}

type Locks []Lock

func (l Locks) Len() int {
	return len(l)
}

func (l Locks) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}

func (l Locks) Less(i, j int) bool {
	if l[i].Name == l[j].Name {
		return l[i].Url < l[j].Url
	}
	return l[i].Name < l[j].Name
}

// parsePackages parses Debian's control file which described packages.
//
// See chapter 5.1 (Syntax of control files) of the Debian Policy Manual:
// http://www.debian.org/doc/debian-policy/ch-controlfields.html
func parsePackages(r io.Reader) ([]map[string]string, error) {
	// Packages are separated by double newline, use scanner to split them.
	scanner := bufio.NewScanner(r)
	scanner.Split(func(data []byte, atEOF bool) (int, []byte, error) {
		const separator = "\n\n"
		if i := bytes.Index(data, []byte(separator)); i != -1 {
			return i + len(separator), data[:i], nil
		}
		return 0, nil, nil
	})
	space := regexp.MustCompile(`\s+`)
	exp := regexp.MustCompile(`(?m)^(?P<key>\S+): (?P<value>(.*)(?:$\s^ .*)*)$`)
	var ps []map[string]string
	for scanner.Scan() {
		p := make(map[string]string)
		for _, m := range exp.FindAllStringSubmatch(scanner.Text(), -1) {
			p[m[1]] = space.ReplaceAllString(m[2], " ")
		}
		ps = append(ps, p)
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return ps, nil
}

func downloadPackageList(config *Config, depends bool) ([]Lock, error) {
	type descriptor struct {
		name    string
		version string
		url     string
		hash    string
		depends []string
	}
	descriptors := map[string]map[string]descriptor{}

	file, err := os.Open(config.Keyring)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	keyring, err := openpgp.ReadKeyRing(file)
	if err != nil {
		return nil, err
	}

	for _, source := range config.Sources {
		sourceUrl, err := url.Parse(source)
		if err != nil {
			return nil, fmt.Errorf("%s: invalid url", source)
		}

		for _, dist := range config.Dists {
			releaseUrl := *sourceUrl
			releaseUrl.Path = path.Join(sourceUrl.Path, "dists", dist, "Release")
			r, err := http.Get(releaseUrl.String())
			if err != nil {
				return nil, err
			}
			if r.StatusCode != http.StatusOK {
				return nil, fmt.Errorf("%s not available", releaseUrl.String())
			}
			defer r.Body.Close()

			b, err := ioutil.ReadAll(r.Body)
			if err != nil {
				return nil, err
			}
			var lines []string
			sha256section := false
			for _, l := range strings.Split(string(b), "\n") {
				if sha256section {
					if strings.HasPrefix(l, " ") {
						lines = append(lines, l[1:])
					} else {
						sha256section = false
					}
				} else if strings.HasPrefix(l, "SHA256:") {
					sha256section = true
				}
			}

			keyUrl := *sourceUrl
			keyUrl.Path = path.Join(keyUrl.Path, "dists", dist, "Release.gpg")
			r, err = http.Get(keyUrl.String())
			if err != nil {
				return nil, fmt.Errorf("failed to download the key: %v", err)
			}
			defer r.Body.Close()

			_, err = openpgp.CheckArmoredDetachedSignature(keyring, bytes.NewReader(b), r.Body)
			if err != nil {
				return nil, fmt.Errorf("%s: failed to check the signature: %v", keyUrl.String(), err)
			}

		Architectures:
			for _, a := range config.Architectures {
				for _, c := range config.Components {
					packagesUrl := *sourceUrl
					packagesUrl.Path = path.Join(packagesUrl.Path, "dists", dist, c, "binary-"+a, packagesFile)
					fmt.Printf("Processing %s %s %s %s...", source, dist, a, c)
					r, err := http.Get(packagesUrl.String())
					if err != nil || r.StatusCode != http.StatusOK {
						fmt.Printf(" skipping\n")
						continue Architectures
					}
					defer r.Body.Close()
					fmt.Printf("\n")

					buf, err := ioutil.ReadAll(r.Body)
					if err != nil {
						return nil, fmt.Errorf("failed to read packages file: %v", err)
					}

					var checksum string
					f := path.Join(c, "binary-"+a, packagesFile)
					for _, l := range lines {
						if strings.HasSuffix(l, f) {
							checksum = strings.Fields(l)[0]
							break
						}
					}
					if checksum == "" {
						return nil, fmt.Errorf("%s: checksum missing", f)
					}

					sum := sha256.Sum256(buf)
					if checksum != hex.EncodeToString(sum[:]) {
						return nil, fmt.Errorf("%s: checksum doesn't match %s vs %s", packagesUrl.String(), checksum, hex.EncodeToString(sum[:]))
					}

					g, err := xz.NewReader(bytes.NewReader(buf))
					if err != nil {
						return nil, err
					}

					ps, err := parsePackages(g)
					if err != nil {
						return nil, err
					}

					// We only want development libraries, filter out everything else.
					for _, p := range ps {
						var include bool
						// Use sections as a coarse grained filter.
						switch strings.TrimPrefix(p["Section"], c+"/") {
						case
							"devel",
							"libdevel",
							"libs",
							"python",
							"x11":
							include = true
						}
						if include {
							if ts, ok := p["Tag"]; ok {
								// Use tags as a more fine-grained filter.
								include = false
								for _, n := range strings.Split(ts, ", ") {
									t := strings.Split(strings.TrimSpace(n), " ")[0]
									switch t {
									case
										"devel::library",
										"role::devel-lib",
										"role::shared-lib",
										"x11::library":
										include = true
									}
								}
							}
						}
						// Skip everything that doesn't match.
						if include {
							var depends []string
							for _, n := range strings.Split(p["Depends"], ", ") {
								depends = append(depends, strings.Split(strings.TrimSpace(n), " ")[0])
							}
							n := p["Package"]
							//fmt.Printf("%s (%s)\n", n, a)
							if _, ok := descriptors[n]; !ok {
								descriptors[n] = map[string]descriptor{}
							}
							url := *sourceUrl
							url.Path = path.Join(url.Path, p["Filename"])
							descriptors[n][a] = descriptor{
								name:    p["Package"],
								version: p["Version"],
								url:     url.String(),
								hash:    p["SHA256"],
								depends: depends,
							}
						}
					}
				}
			}
		}
	}

	type dependency struct {
		name         string
		architecture string
	}

	// Place the initial set of packages into queue.
	var queue []dependency
	for _, p := range config.Packages {
		if len(p.Architectures) > 0 {
			for _, a := range p.Architectures {
				queue = append(queue, dependency{
					name:         p.Name,
					architecture: a,
				})
			}
		} else {
			for _, a := range config.Architectures {
				queue = append(queue, dependency{
					name:         p.Name,
					architecture: a,
				})
			}
		}
	}

	// Process all dependencies until we drain the queue.
	locks := map[string]map[string]Lock{}
	for len(queue) > 0 {
		p := queue[0]
		queue = queue[1:]
		if lock, ok := locks[p.name]; ok {
			if _, ok := lock[p.architecture]; ok {
				continue
			}
		}
		if ds, ok := descriptors[p.name]; ok {
			if _, ok := locks[p.name]; !ok {
				locks[p.name] = map[string]Lock{}
			}
			if d, ok := ds[p.architecture]; ok {
				locks[p.name][p.architecture] = Lock{
					Name:    d.name,
					Version: d.version,
					Url:     d.url,
					Hash:    d.hash,
				}
				if depends {
					for _, n := range d.depends {
						queue = append(queue, dependency{
							name:         n,
							architecture: p.architecture,
						})
					}
				}
			} else {
				return nil, fmt.Errorf("package %q not found for architecture %q", p.name, p.architecture)
			}
		} else {
			return nil, fmt.Errorf("package %q not found", p.name)
		}
	}

	// Eliminate all duplicates.
	hashes := map[string]Lock{}
	for _, l := range locks {
		for _, p := range l {
			hashes[p.Hash] = p
		}
	}

	// Flatten into a list.
	var list []Lock
	for _, p := range hashes {
		list = append(list, p)
	}

	return list, nil
}

func relativize(link, target, dir string, patterns []string) error {
	for _, p := range patterns {
		matches, err := filepath.Glob(filepath.Join(dir, p))
		if err != nil {
			return err
		}
		for _, m := range matches {
			if link == m {
				if err := os.Remove(link); err != nil {
					return err
				}
				relDir := ".." + strings.Repeat("/..", strings.Count(p, "/")-1)
				if err := os.Symlink(relDir+target, link); err != nil {
					return err
				}
				return nil
			}
		}
	}
	return nil
}

func installSysroot(list []Lock, installDir, debsCache string) error {
	if err := os.MkdirAll(debsCache, 0777); err != nil {
		return err
	}

	if err := os.RemoveAll(installDir); err != nil {
		return err
	}

	// This is only needed when running dpkg-shlibdeps.
	if err := os.MkdirAll(filepath.Join(installDir, "debian"), 0777); err != nil {
		return err
	}

	// An empty control file is necessary to run dpkg-shlibdeps.
	if file, err := os.OpenFile(filepath.Join(installDir, "debian", "control"), os.O_RDONLY|os.O_CREATE, 0644); err != nil {
		return err
	} else {
		file.Close()
	}

	for _, pkg := range list {
		u, err := url.Parse(pkg.Url)
		if err != nil {
			return err
		}

		filename := filepath.Base(u.Path)
		deb := filepath.Join(debsCache, filename)
		if _, err := os.Stat(deb); os.IsNotExist(err) {
			fmt.Printf("Downloading %s...\n", filename)

			r, err := http.Get(u.String())
			if err != nil {
				return err
			}
			defer r.Body.Close()

			buf, err := ioutil.ReadAll(r.Body)
			if err != nil {
				return err
			}

			sum := sha256.Sum256(buf)
			if pkg.Hash != hex.EncodeToString(sum[:]) {
				return fmt.Errorf("%s: checksum doesn't match", filename)
			}

			if err := ioutil.WriteFile(deb, buf, 0644); err != nil {
				return err
			}
		}

		fmt.Printf("Installing %s...\n", filename)
		// Extract the content of the package into the install directory.
		if err := exec.Command("dpkg-deb", "-x", deb, installDir).Run(); err != nil {
			return err
		}
		// Get the package name.
		cmd := exec.Command("dpkg-deb", "--field", deb, "Package")
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			return err
		}
		if err := cmd.Start(); err != nil {
			return err
		}
		r := bufio.NewReader(stdout)
		baseDir, _, err := r.ReadLine()
		if err != nil {
			return err
		}
		if err := cmd.Wait(); err != nil {
			return err
		}
		// Construct the path which contains the control information files.
		controlDir := filepath.Join(installDir, "debian", string(baseDir), "DEBIAN")
		if err := os.MkdirAll(controlDir, 0777); err != nil {
			return err
		}
		// Extract the control information files.
		err = exec.Command("dpkg-deb", "-e", deb, controlDir).Run()
		if err != nil {
			return err
		}
	}

	// Prune /usr/share, leave only pkgconfig files.
	files, err := ioutil.ReadDir(filepath.Join(installDir, "usr", "share"))
	if err != nil {
		return err
	}
	for _, file := range files {
		if file.Name() != "pkgconfig" {
			if err := os.RemoveAll(filepath.Join(installDir, "usr", "share", file.Name())); err != nil {
				return err
			}
		}
	}

	// Ensure that we don't have duplicate file names that only differ in case.
	type rename struct{ oldpath, newpath string }
	renames := []rename{}

	for _, d := range []string{"usr/include/linux"} {
		p := filepath.Join(installDir, d)
		if _, err := os.Stat(p); os.IsNotExist(err) {
			continue
		}
		paths := make(map[string][]string)
		if err := filepath.Walk(p, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if info.Mode().IsRegular() && filepath.Ext(path) == ".h" {
				name := strings.ToLower(path)
				if _, ok := paths[name]; !ok {
					paths[name] = []string{}
				}
				paths[name] = append(paths[name], path)
			}
			return nil
		}); err != nil {
			return err
		}

		for _, ps := range paths {
			if len(ps) > 1 {
				sort.Sort(sort.Reverse(sort.StringSlice(ps)))
				for i, p := range ps {
					if i > 0 {
						ext := filepath.Ext(p)
						renames = append(renames, rename{p, p[:len(p)-len(ext)] + strings.Repeat("_", i) + ext})
					}
				}
			}
		}
	}

	for _, r := range renames {
		fmt.Printf("Renaming %s to %s\n", r.oldpath, r.newpath)
		if err := os.Rename(r.oldpath, r.newpath); err != nil {
			return err
		}
	}

	for _, d := range []string{"usr/include/linux"} {
		p := filepath.Join(installDir, d)
		if _, err := os.Stat(p); os.IsNotExist(err) {
			continue
		}
		if err := filepath.Walk(p, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			update := false
			if info.Mode().IsRegular() && filepath.Ext(path) == ".h" {
				content, err := ioutil.ReadFile(path)
				if err != nil {
					return err
				}
				for _, r := range renames {
					oldbase := filepath.Base(r.oldpath)
					newbase := filepath.Base(r.newpath)
					if strings.Contains(string(content), oldbase) {
						content = bytes.Replace(content, []byte(oldbase), []byte(newbase), 1)
						update = true
					}
				}
				if update {
					fmt.Printf("Updating %s...\n", path)
					if err := ioutil.WriteFile(path, content, info.Mode()); err != nil {
						return err
					}
				}
			}
			return nil
		}); err != nil {
			return err
		}
	}

	// Relativize all symlinks within the sysroot.
	for _, d := range []string{"usr/lib", "lib64", "lib"} {
		p := filepath.Join(installDir, d)
		if _, err := os.Stat(p); os.IsNotExist(err) {
			continue
		}
		if err := filepath.Walk(p, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if info.Mode()&os.ModeSymlink == os.ModeSymlink {
				target, err := os.Readlink(path)
				if err != nil {
					return err
				}
				if !filepath.IsAbs(target) {
					return nil
				}
				patterns := []string{
					"usr/lib/gcc/*-linux-gnu*/*/*",
					"usr/lib/*-linux-gnu*/*",
					"usr/lib/*",
					"lib64/*",
					"lib/*",
				}
				if err := relativize(path, target, installDir, patterns); err != nil {
					return err
				}
				if _, err := os.Stat(path); os.IsNotExist(err) {
					return fmt.Errorf("%s: broken link", path)
				}
			}
			return nil
		}); err != nil {
			return err
		}
	}

	// Rewrite and relativize all linkerscripts.
	linkerscripts := []string{
		"usr/lib/*-linux-gnu*/libpthread.so",
		"usr/lib/*-linux-gnu*/libc.so",
	}
	for _, l := range linkerscripts {
		matches, err := filepath.Glob(filepath.Join(installDir, l))
		if err != nil {
			return err
		}
		for _, path := range matches {
			read, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}
			sub := regexp.MustCompile(`(/usr)?/lib/[a-z0-9_]+-linux-gnu[a-z0-9]*/`)
			contents := sub.ReplaceAllString(string(read), "")
			if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
				return err
			}
		}
	}

	if err := os.RemoveAll(filepath.Join(installDir, "debian")); err != nil {
		return err
	}

	return nil
}

type updateCmd struct {
	config   string
	lockfile string
	depends  bool
}

func (*updateCmd) Name() string     { return "update" }
func (*updateCmd) Synopsis() string { return "Update the lock file." }
func (*updateCmd) Usage() string {
	return `update [-config] [-lock] [-depends]:
	Update the lock file to include specific package versions.
`
}

func (c *updateCmd) SetFlags(f *flag.FlagSet) {
	f.StringVar(&c.config, "config", "packages.yml", "Package configuration")
	f.StringVar(&c.lockfile, "lock", "packages.lock", "Lockfile filename")
	f.BoolVar(&c.depends, "depends", false, "Transitively include dependencies")
}

func (c *updateCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	d, err := ioutil.ReadFile(c.config)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to read config: %v\n", err)
		return subcommands.ExitFailure
	}
	config := &Config{}
	if err := yaml.Unmarshal(d, &config); err != nil {
		fmt.Fprintf(os.Stderr, "failed to unmarshal config: %v\n", err)
		return subcommands.ExitFailure
	}

	if _, err := os.Stat(config.Keyring); os.IsNotExist(err) {
		fmt.Fprintf(os.Stderr, "keyring file '%s' missing\n", config.Keyring)
		return subcommands.ExitUsageError
	}

	list, err := downloadPackageList(config, c.depends)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to download package list: %v\n", err)
		return subcommands.ExitFailure
	}
	sort.Sort(Locks(list))

	hash := sha256.New()
	hash.Write(d)

	lockfile := Lockfile{
		Updated:  time.Now(),
		Hash:     fmt.Sprintf("%x", hash.Sum(nil)),
		Packages: list,
	}

	l, err := yaml.Marshal(&lockfile)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to marshal lockfile: %v\n", err)
		return subcommands.ExitFailure
	}

	if err := ioutil.WriteFile(c.lockfile, l, 0666); err != nil {
		fmt.Fprintf(os.Stderr, "failed to write lockfile: %v\n", err)
		return subcommands.ExitFailure
	}

	return subcommands.ExitSuccess
}

type installCmd struct {
	outDir    string
	debsCache string
}

func (*installCmd) Name() string     { return "install" }
func (*installCmd) Synopsis() string { return "Install packages" }
func (*installCmd) Usage() string {
	return `install [-out] [-cache] <lockfile>:
  Install the specific versions from the lock file.
`
}

func (c *installCmd) SetFlags(f *flag.FlagSet) {
	f.StringVar(&c.outDir, "out", "sysroot", "Output directory")
	f.StringVar(&c.debsCache, "cache", "debs", "Cache for .deb files")
}

func (c *installCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	if len(f.Args()) != 1 {
		fmt.Fprintln(os.Stderr, "missing lockfile argument")
		return subcommands.ExitUsageError
	}
	lockfile := f.Args()[0]

	d, err := ioutil.ReadFile(lockfile)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to read lockfile: %v\n", err)
		return subcommands.ExitFailure
	}
	var lock Lockfile
	if err := yaml.Unmarshal(d, &lock); err != nil {
		fmt.Fprintf(os.Stderr, "failed to unmarshal lockfile: %v\n", err)
		return subcommands.ExitFailure
	}

	if err := installSysroot(lock.Packages, c.outDir, c.debsCache); err != nil {
		fmt.Fprintf(os.Stderr, "failed to install sysroot: %v\n", err)
		return subcommands.ExitFailure
	}
	return subcommands.ExitSuccess
}

func main() {
	subcommands.Register(subcommands.HelpCommand(), "")
	subcommands.Register(subcommands.FlagsCommand(), "")
	subcommands.Register(subcommands.CommandsCommand(), "")
	subcommands.Register(&updateCmd{}, "")
	subcommands.Register(&installCmd{}, "")

	flag.Parse()
	ctx := context.Background()
	os.Exit(int(subcommands.Execute(ctx)))
}