blob: d8d1552165961d996e260ab10045e474369daef9 [file] [log] [blame]
// Copyright 2017 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Generates a Debian based sysroot.
package main
import (
"bufio"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
"github.com/google/subcommands"
"golang.org/x/crypto/openpgp"
"gopkg.in/yaml.v2"
)
// gpg keyring file generated using:
// export KEYS="46925553 2B90D010 518E17E1 1A7B6500"
// gpg --keyserver keys.gnupg.net --recv-keys $KEYS
// gpg --output ./debian-archive-stretch-stable.gpg --export $KEYS
const (
// http://http.us.debian.org/debian no longer contains arm64 packages
// for jessie so we instead use the last snapshot that still has them.
aptRepo = "https://snapshot.debian.org/archive/debian/20190324T093412Z"
)
type stringsValue []string
func (i *stringsValue) String() string {
return strings.Join(*i, ",")
}
func (i *stringsValue) Set(value string) error {
*i = strings.Split(value, ",")
return nil
}
type Config struct {
Dists []string `yaml:"dists"`
Components []string `yaml:"components"`
Keyring string `yaml:"keyring"`
Architectures []string `yaml:"architectures"`
Packages []Package `yaml:"packages"`
}
type Package struct {
Name string `yaml:"package"`
Architectures []string `yaml:"architectures,omitempty"`
}
type Lockfile struct {
Hash string `yaml:"hash"`
Updated time.Time `yaml:"updated"`
Packages []Lock `yaml:"packages"`
}
type Lock struct {
Name string `yaml:"package"`
Version string `yaml:"version"`
Filename string `yaml:"filename"`
Hash string `yaml:"hash"`
}
type Locks []Lock
func (l Locks) Len() int {
return len(l)
}
func (l Locks) Swap(i, j int) {
l[i], l[j] = l[j], l[i]
}
func (l Locks) Less(i, j int) bool {
if l[i].Name == l[j].Name {
return l[i].Filename < l[j].Filename
}
return l[i].Name < l[j].Name
}
// Descriptor represents a Debian package description.
type Descriptor map[string]string
// parsePackages parses Debian's control file which described packages.
//
// See chapter 5.1 (Syntax of control files) of the Debian Policy Manual:
// http://www.debian.org/doc/debian-policy/ch-controlfields.html
func parsePackages(r io.Reader) ([]Descriptor, error) {
// Packages are separated by double newline, use scanner to split them.
scanner := bufio.NewScanner(r)
scanner.Split(func(data []byte, atEOF bool) (int, []byte, error) {
const separator = "\n\n"
if i := bytes.Index(data, []byte(separator)); i != -1 {
return i + len(separator), data[:i], nil
}
return 0, nil, nil
})
exp := regexp.MustCompile(`(?m)^(?P<key>\S+): (?P<value>(.*)(?:$\s^ .*)*)$`)
var descriptors []Descriptor
for scanner.Scan() {
descriptor := make(Descriptor)
for _, m := range exp.FindAllStringSubmatch(scanner.Text(), -1) {
descriptor[m[1]] = m[2]
}
descriptors = append(descriptors, descriptor)
}
if err := scanner.Err(); err != nil {
return nil, err
}
return descriptors, nil
}
func downloadPackageList(config *Config, depends bool) ([]Lock, error) {
descriptors := map[string]map[string]Descriptor{}
file, err := os.Open(config.Keyring)
if err != nil {
return nil, err
}
defer file.Close()
keyring, err := openpgp.ReadKeyRing(file)
if err != nil {
return nil, err
}
for _, dist := range config.Dists {
u, err := url.Parse(aptRepo)
u.Path = path.Join(u.Path, "dists", dist, "Release")
r, err := http.Get(u.String())
if err != nil {
return nil, err
}
defer r.Body.Close()
b, err := ioutil.ReadAll(r.Body)
if err != nil {
return nil, err
}
var lines []string
sha256section := false
for _, l := range strings.Split(string(b), "\n") {
if sha256section {
if strings.HasPrefix(l, " ") {
lines = append(lines, l[1:])
} else {
sha256section = false
}
} else if strings.HasPrefix(l, "SHA256:") {
sha256section = true
}
}
u, err = url.Parse(aptRepo)
u.Path = path.Join(u.Path, "dists", dist, "Release.gpg")
r, err = http.Get(u.String())
if err != nil {
return nil, err
}
defer r.Body.Close()
_, err = openpgp.CheckArmoredDetachedSignature(keyring, bytes.NewReader(b), r.Body)
if err != nil {
return nil, err
}
for _, a := range config.Architectures {
for _, c := range config.Components {
u, err := url.Parse(aptRepo)
u.Path = path.Join(u.Path, "dists", dist, c, "binary-"+a, "Packages.gz")
r, err := http.Get(u.String())
if err != nil {
return nil, err
}
defer r.Body.Close()
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return nil, err
}
var checksum string
f := path.Join(c, "binary-"+a, "Packages.gz")
for _, l := range lines {
if strings.HasSuffix(l, f) {
checksum = strings.Fields(l)[0]
break
}
}
if checksum == "" {
return nil, fmt.Errorf("%s: checksum missing", f)
}
sum := sha256.Sum256(buf)
if checksum != hex.EncodeToString(sum[:]) {
return nil, fmt.Errorf("%s: checksum doesn't match", f)
}
g, err := gzip.NewReader(bytes.NewReader(buf))
if err != nil {
return nil, err
}
ps, err := parsePackages(g)
if err != nil {
return nil, err
}
// We only want development libraries, filter out everything else.
for _, p := range ps {
// Use sections as a coarse grained filter.
var section bool
switch p["Section"] {
case "libs", "libdevel", "devel", "x11":
section = true
}
// Use tags as a more fine-grained filter.
var tag bool
for _, n := range strings.Split(p["Tag"], ", ") {
t := strings.Split(strings.TrimSpace(n), " ")[0]
switch t {
case "devel::library", "x11::library", "role::devel-lib", "role::shared-lib":
tag = true
}
}
// Skip everything that doesn't match.
if section && tag {
n := p["Package"]
if _, ok := descriptors[n]; !ok {
descriptors[n] = map[string]Descriptor{}
}
descriptors[n][a] = p
}
}
}
}
}
type dependency struct {
name string
architecture string
}
// Place the initial set of packages into queue.
var queue []dependency
for _, p := range config.Packages {
if len(p.Architectures) > 0 {
for _, a := range p.Architectures {
queue = append(queue, dependency{
name: p.Name,
architecture: a,
})
}
} else {
for _, a := range config.Architectures {
queue = append(queue, dependency{
name: p.Name,
architecture: a,
})
}
}
}
// Process all dependencies until we drain the queue.
locks := map[string]map[string]Lock{}
for len(queue) > 0 {
p := queue[0]
queue = queue[1:]
if lock, ok := locks[p.name]; ok {
if _, ok := lock[p.architecture]; ok {
continue
}
}
if ds, ok := descriptors[p.name]; ok {
if _, ok := locks[p.name]; !ok {
locks[p.name] = map[string]Lock{}
}
if pkg, ok := ds[p.architecture]; ok {
locks[p.name][p.architecture] = Lock{
Name: pkg["Package"],
Version: pkg["Version"],
Filename: pkg["Filename"],
Hash: pkg["SHA256"],
}
if depends {
for _, n := range strings.Split(pkg["Depends"], ", ") {
d := strings.Split(strings.TrimSpace(n), " ")[0]
queue = append(queue, dependency{
name: d,
architecture: p.architecture,
})
}
}
} else {
return nil, fmt.Errorf("package %q not found for architecture %q", p.name, p.architecture)
}
} else {
return nil, fmt.Errorf("package %q not found", p.name)
}
}
// Eliminate all duplicates.
hashes := map[string]Lock{}
for _, l := range locks {
for _, p := range l {
hashes[p.Hash] = p
}
}
// Flatten into a list.
var list []Lock
for _, p := range hashes {
list = append(list, p)
}
return list, nil
}
func relativize(link, target, dir string, patterns []string) error {
for _, p := range patterns {
matches, err := filepath.Glob(filepath.Join(dir, p))
if err != nil {
return err
}
for _, m := range matches {
if link == m {
if err := os.Remove(link); err != nil {
return err
}
relDir := ".." + strings.Repeat("/..", strings.Count(p, "/")-1)
if err := os.Symlink(relDir+target, link); err != nil {
return err
}
return nil
}
}
}
return nil
}
func installSysroot(list []Lock, installDir, debsCache string) error {
if err := os.MkdirAll(debsCache, 0777); err != nil {
return err
}
if err := os.RemoveAll(installDir); err != nil {
return err
}
// This is only needed when running dpkg-shlibdeps.
if err := os.MkdirAll(filepath.Join(installDir, "debian"), 0777); err != nil {
return err
}
// An empty control file is necessary to run dpkg-shlibdeps.
if file, err := os.OpenFile(filepath.Join(installDir, "debian", "control"), os.O_RDONLY|os.O_CREATE, 0644); err != nil {
return err
} else {
file.Close()
}
for _, pkg := range list {
filename := filepath.Base(pkg.Filename)
deb := filepath.Join(debsCache, filename)
if _, err := os.Stat(deb); os.IsNotExist(err) {
fmt.Printf("Downloading %s...\n", filename)
u, err := url.Parse(aptRepo)
u.Path = path.Join(u.Path, pkg.Filename)
r, err := http.Get(u.String())
if err != nil {
return err
}
defer r.Body.Close()
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return err
}
sum := sha256.Sum256(buf)
if pkg.Hash != hex.EncodeToString(sum[:]) {
return fmt.Errorf("%s: checksum doesn't match", filename)
}
if err := ioutil.WriteFile(deb, buf, 0644); err != nil {
return err
}
}
fmt.Printf("Installing %s...\n", filename)
// Extract the content of the package into the install directory.
err := exec.Command("dpkg-deb", "-x", deb, installDir).Run()
if err != nil {
return err
}
// Get the package name.
cmd := exec.Command("dpkg-deb", "--field", deb, "Package")
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
if err := cmd.Start(); err != nil {
return err
}
r := bufio.NewReader(stdout)
baseDir, _, err := r.ReadLine()
if err != nil {
return err
}
if err := cmd.Wait(); err != nil {
return err
}
// Construct the path which contains the control information files.
controlDir := filepath.Join(installDir, "debian", string(baseDir), "DEBIAN")
if err := os.MkdirAll(controlDir, 0777); err != nil {
return err
}
// Extract the control information files.
err = exec.Command("dpkg-deb", "-e", deb, controlDir).Run()
if err != nil {
return err
}
}
// Prune /usr/share, leave only pkgconfig files.
files, err := ioutil.ReadDir(filepath.Join(installDir, "usr", "share"))
if err != nil {
return err
}
for _, file := range files {
if file.Name() != "pkgconfig" {
if err := os.RemoveAll(filepath.Join(installDir, "usr", "share", file.Name())); err != nil {
return err
}
}
}
// Ensure that we don't have duplicate file names that only differ in case.
type rename struct{ oldpath, newpath string }
renames := []rename{}
for _, d := range []string{"usr/include/linux"} {
p := filepath.Join(installDir, d)
if _, err := os.Stat(p); os.IsNotExist(err) {
continue
}
paths := make(map[string][]string)
if err := filepath.Walk(p, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode().IsRegular() && filepath.Ext(path) == ".h" {
name := strings.ToLower(path)
if _, ok := paths[name]; !ok {
paths[name] = []string{}
}
paths[name] = append(paths[name], path)
}
return nil
}); err != nil {
return err
}
for _, ps := range paths {
if len(ps) > 1 {
sort.Sort(sort.Reverse(sort.StringSlice(ps)))
for i, p := range ps {
if i > 0 {
ext := filepath.Ext(p)
renames = append(renames, rename{p, p[:len(p)-len(ext)] + strings.Repeat("_", i) + ext})
}
}
}
}
}
for _, r := range renames {
fmt.Printf("Renaming %s to %s\n", r.oldpath, r.newpath)
if err := os.Rename(r.oldpath, r.newpath); err != nil {
return err
}
}
for _, d := range []string{"usr/include/linux"} {
p := filepath.Join(installDir, d)
if _, err := os.Stat(p); os.IsNotExist(err) {
continue
}
if err := filepath.Walk(p, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
update := false
if info.Mode().IsRegular() && filepath.Ext(path) == ".h" {
content, err := ioutil.ReadFile(path)
if err != nil {
return err
}
for _, r := range renames {
oldbase := filepath.Base(r.oldpath)
newbase := filepath.Base(r.newpath)
if strings.Contains(string(content), oldbase) {
content = bytes.Replace(content, []byte(oldbase), []byte(newbase), 1)
update = true
}
}
if update {
fmt.Printf("Updating %s...\n", path)
if err := ioutil.WriteFile(path, content, info.Mode()); err != nil {
return err
}
}
}
return nil
}); err != nil {
return err
}
}
// Relativize all symlinks within the sysroot.
for _, d := range []string{"usr/lib", "lib64", "lib"} {
p := filepath.Join(installDir, d)
if _, err := os.Stat(p); os.IsNotExist(err) {
continue
}
if err := filepath.Walk(p, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
target, err := os.Readlink(path)
if err != nil {
return err
}
if !filepath.IsAbs(target) {
return nil
}
patterns := []string{
"usr/lib/gcc/*-linux-gnu*/*/*",
"usr/lib/*-linux-gnu*/*",
"usr/lib/*",
"lib64/*",
"lib/*",
}
if err := relativize(path, target, installDir, patterns); err != nil {
return err
}
if _, err := os.Stat(path); os.IsNotExist(err) {
return fmt.Errorf("%s: broken link", path)
}
}
return nil
}); err != nil {
return err
}
}
// Rewrite and relativize all linkerscripts.
linkerscripts := []string{
"usr/lib/*-linux-gnu*/libpthread.so",
"usr/lib/*-linux-gnu*/libc.so",
}
for _, l := range linkerscripts {
matches, err := filepath.Glob(filepath.Join(installDir, l))
if err != nil {
return err
}
for _, path := range matches {
read, err := ioutil.ReadFile(path)
if err != nil {
return err
}
sub := regexp.MustCompile(`(/usr)?/lib/[a-z0-9_]+-linux-gnu[a-z0-9]*/`)
contents := sub.ReplaceAllString(string(read), "")
if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
return err
}
}
}
if err := os.RemoveAll(filepath.Join(installDir, "debian")); err != nil {
return err
}
return nil
}
type updateCmd struct {
config string
lockfile string
depends bool
}
func (*updateCmd) Name() string { return "update" }
func (*updateCmd) Synopsis() string { return "Update the lock file." }
func (*updateCmd) Usage() string {
return `update [-config] [-lock] [-depends]:
Update the lock file to include specific package versions.
`
}
func (c *updateCmd) SetFlags(f *flag.FlagSet) {
f.StringVar(&c.config, "config", "packages.yml", "Package configuration")
f.StringVar(&c.lockfile, "lock", "packages.lock", "Lockfile filename")
f.BoolVar(&c.depends, "depends", false, "Transitively include dependencies")
}
func (c *updateCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
d, err := ioutil.ReadFile(c.config)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to read config: %v\n", err)
return subcommands.ExitFailure
}
config := &Config{}
if err := yaml.Unmarshal(d, &config); err != nil {
fmt.Fprintf(os.Stderr, "failed to unmarshal config: %v\n", err)
return subcommands.ExitFailure
}
if _, err := os.Stat(config.Keyring); os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "keyring file '%s' missing\n", config.Keyring)
return subcommands.ExitUsageError
}
list, err := downloadPackageList(config, c.depends)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to download package list: %v\n", err)
return subcommands.ExitFailure
}
sort.Sort(Locks(list))
hash := sha256.New()
hash.Write(d)
lockfile := Lockfile{
Updated: time.Now(),
Hash: fmt.Sprintf("%x", hash.Sum(nil)),
Packages: list,
}
l, err := yaml.Marshal(&lockfile)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to marshal lockfile: %v\n", err)
return subcommands.ExitFailure
}
if err := ioutil.WriteFile(c.lockfile, l, 0666); err != nil {
fmt.Fprintf(os.Stderr, "failed to write lockfile: %v\n", err)
return subcommands.ExitFailure
}
return subcommands.ExitSuccess
}
type installCmd struct {
outDir string
debsCache string
}
func (*installCmd) Name() string { return "install" }
func (*installCmd) Synopsis() string { return "Install packages" }
func (*installCmd) Usage() string {
return `install [-out] [-cache] <lockfile>:
Install the specific versions from the lock file.
`
}
func (c *installCmd) SetFlags(f *flag.FlagSet) {
f.StringVar(&c.outDir, "out", "sysroot", "Output directory")
f.StringVar(&c.debsCache, "cache", "debs", "Cache for .deb files")
}
func (c *installCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
if len(f.Args()) != 1 {
fmt.Fprintln(os.Stderr, "missing lockfile argument")
return subcommands.ExitUsageError
}
lockfile := f.Args()[0]
d, err := ioutil.ReadFile(lockfile)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to read lockfile: %v\n", err)
return subcommands.ExitFailure
}
var lock Lockfile
if err := yaml.Unmarshal(d, &lock); err != nil {
fmt.Fprintf(os.Stderr, "failed to unmarshal lockfile: %v\n", err)
return subcommands.ExitFailure
}
if err := installSysroot(lock.Packages, c.outDir, c.debsCache); err != nil {
fmt.Fprintf(os.Stderr, "failed to install sysroot: %v\n", err)
return subcommands.ExitFailure
}
return subcommands.ExitSuccess
}
func main() {
subcommands.Register(subcommands.HelpCommand(), "")
subcommands.Register(subcommands.FlagsCommand(), "")
subcommands.Register(subcommands.CommandsCommand(), "")
subcommands.Register(&updateCmd{}, "")
subcommands.Register(&installCmd{}, "")
flag.Parse()
ctx := context.Background()
os.Exit(int(subcommands.Execute(ctx)))
}