blob: 6943a7f8bc0c1b273949387686e1c39c5820982c [file] [log] [blame]
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package archive
import (
"archive/tar"
"context"
"encoding/json"
"fmt"
"io"
"path"
"sort"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
type exportOptions struct {
manifests []ocispec.Descriptor
platform platforms.MatchComparer
allPlatforms bool
skipDockerManifest bool
blobRecordOptions blobRecordOptions
}
// ExportOpt defines options for configuring exported descriptors
type ExportOpt func(context.Context, *exportOptions) error
// WithPlatform defines the platform to require manifest lists have
// not exporting all platforms.
// Additionally, platform is used to resolve image configs for
// Docker v1.1, v1.2 format compatibility.
func WithPlatform(p platforms.MatchComparer) ExportOpt {
return func(ctx context.Context, o *exportOptions) error {
o.platform = p
return nil
}
}
// WithAllPlatforms exports all manifests from a manifest list.
// Missing content will fail the export.
func WithAllPlatforms() ExportOpt {
return func(ctx context.Context, o *exportOptions) error {
o.allPlatforms = true
return nil
}
}
// WithSkipDockerManifest skips creation of the Docker compatible
// manifest.json file.
func WithSkipDockerManifest() ExportOpt {
return func(ctx context.Context, o *exportOptions) error {
o.skipDockerManifest = true
return nil
}
}
// WithImage adds the provided images to the exported archive.
func WithImage(is images.Store, name string) ExportOpt {
return func(ctx context.Context, o *exportOptions) error {
img, err := is.Get(ctx, name)
if err != nil {
return err
}
img.Target.Annotations = addNameAnnotation(name, img.Target.Annotations)
o.manifests = append(o.manifests, img.Target)
return nil
}
}
// WithManifest adds a manifest to the exported archive.
// When names are given they will be set on the manifest in the
// exported archive, creating an index record for each name.
// When no names are provided, it is up to caller to put name annotation to
// on the manifest descriptor if needed.
func WithManifest(manifest ocispec.Descriptor, names ...string) ExportOpt {
return func(ctx context.Context, o *exportOptions) error {
if len(names) == 0 {
o.manifests = append(o.manifests, manifest)
}
for _, name := range names {
mc := manifest
mc.Annotations = addNameAnnotation(name, manifest.Annotations)
o.manifests = append(o.manifests, mc)
}
return nil
}
}
// BlobFilter returns false if the blob should not be included in the archive.
type BlobFilter func(ocispec.Descriptor) bool
// WithBlobFilter specifies BlobFilter.
func WithBlobFilter(f BlobFilter) ExportOpt {
return func(ctx context.Context, o *exportOptions) error {
o.blobRecordOptions.blobFilter = f
return nil
}
}
// WithSkipNonDistributableBlobs excludes non-distributable blobs such as Windows base layers.
func WithSkipNonDistributableBlobs() ExportOpt {
f := func(desc ocispec.Descriptor) bool {
return !images.IsNonDistributable(desc.MediaType)
}
return WithBlobFilter(f)
}
func addNameAnnotation(name string, base map[string]string) map[string]string {
annotations := map[string]string{}
for k, v := range base {
annotations[k] = v
}
annotations[images.AnnotationImageName] = name
annotations[ocispec.AnnotationRefName] = ociReferenceName(name)
return annotations
}
// Export implements Exporter.
func Export(ctx context.Context, store content.Provider, writer io.Writer, opts ...ExportOpt) error {
var eo exportOptions
for _, opt := range opts {
if err := opt(ctx, &eo); err != nil {
return err
}
}
records := []tarRecord{
ociLayoutFile(""),
ociIndexRecord(eo.manifests),
}
algorithms := map[string]struct{}{}
dManifests := map[digest.Digest]*exportManifest{}
resolvedIndex := map[digest.Digest]digest.Digest{}
for _, desc := range eo.manifests {
switch desc.MediaType {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
mt, ok := dManifests[desc.Digest]
if !ok {
// TODO(containerd): Skip if already added
r, err := getRecords(ctx, store, desc, algorithms, &eo.blobRecordOptions)
if err != nil {
return err
}
records = append(records, r...)
mt = &exportManifest{
manifest: desc,
}
dManifests[desc.Digest] = mt
}
name := desc.Annotations[images.AnnotationImageName]
if name != "" {
mt.names = append(mt.names, name)
}
case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
d, ok := resolvedIndex[desc.Digest]
if !ok {
if err := desc.Digest.Validate(); err != nil {
return err
}
records = append(records, blobRecord(store, desc, &eo.blobRecordOptions))
p, err := content.ReadBlob(ctx, store, desc)
if err != nil {
return err
}
var index ocispec.Index
if err := json.Unmarshal(p, &index); err != nil {
return err
}
var manifests []ocispec.Descriptor
for _, m := range index.Manifests {
if eo.platform != nil {
if m.Platform == nil || eo.platform.Match(*m.Platform) {
manifests = append(manifests, m)
} else if !eo.allPlatforms {
continue
}
}
r, err := getRecords(ctx, store, m, algorithms, &eo.blobRecordOptions)
if err != nil {
return err
}
records = append(records, r...)
}
if len(manifests) >= 1 {
if len(manifests) > 1 {
sort.SliceStable(manifests, func(i, j int) bool {
if manifests[i].Platform == nil {
return false
}
if manifests[j].Platform == nil {
return true
}
return eo.platform.Less(*manifests[i].Platform, *manifests[j].Platform)
})
}
d = manifests[0].Digest
dManifests[d] = &exportManifest{
manifest: manifests[0],
}
} else if eo.platform != nil {
return fmt.Errorf("no manifest found for platform: %w", errdefs.ErrNotFound)
}
resolvedIndex[desc.Digest] = d
}
if d != "" {
if name := desc.Annotations[images.AnnotationImageName]; name != "" {
mt := dManifests[d]
mt.names = append(mt.names, name)
}
}
default:
return fmt.Errorf("only manifests may be exported: %w", errdefs.ErrInvalidArgument)
}
}
if !eo.skipDockerManifest && len(dManifests) > 0 {
tr, err := manifestsRecord(ctx, store, dManifests)
if err != nil {
return fmt.Errorf("unable to create manifests file: %w", err)
}
records = append(records, tr)
}
if len(algorithms) > 0 {
records = append(records, directoryRecord("blobs/", 0755))
for alg := range algorithms {
records = append(records, directoryRecord("blobs/"+alg+"/", 0755))
}
}
tw := tar.NewWriter(writer)
defer tw.Close()
return writeTar(ctx, tw, records)
}
func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descriptor, algorithms map[string]struct{}, brOpts *blobRecordOptions) ([]tarRecord, error) {
var records []tarRecord
exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
if err := desc.Digest.Validate(); err != nil {
return nil, err
}
records = append(records, blobRecord(store, desc, brOpts))
algorithms[desc.Digest.Algorithm().String()] = struct{}{}
return nil, nil
}
childrenHandler := images.ChildrenHandler(store)
handlers := images.Handlers(
childrenHandler,
images.HandlerFunc(exportHandler),
)
// Walk sequentially since the number of fetches is likely one and doing in
// parallel requires locking the export handler
if err := images.Walk(ctx, handlers, desc); err != nil {
return nil, err
}
return records, nil
}
type tarRecord struct {
Header *tar.Header
CopyTo func(context.Context, io.Writer) (int64, error)
}
type blobRecordOptions struct {
blobFilter BlobFilter
}
func blobRecord(cs content.Provider, desc ocispec.Descriptor, opts *blobRecordOptions) tarRecord {
if opts != nil && opts.blobFilter != nil && !opts.blobFilter(desc) {
return tarRecord{}
}
path := path.Join("blobs", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
return tarRecord{
Header: &tar.Header{
Name: path,
Mode: 0444,
Size: desc.Size,
Typeflag: tar.TypeReg,
},
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
r, err := cs.ReaderAt(ctx, desc)
if err != nil {
return 0, fmt.Errorf("failed to get reader: %w", err)
}
defer r.Close()
// Verify digest
dgstr := desc.Digest.Algorithm().Digester()
n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r))
if err != nil {
return 0, fmt.Errorf("failed to copy to tar: %w", err)
}
if dgstr.Digest() != desc.Digest {
return 0, fmt.Errorf("unexpected digest %s copied", dgstr.Digest())
}
return n, nil
},
}
}
func directoryRecord(name string, mode int64) tarRecord {
return tarRecord{
Header: &tar.Header{
Name: name,
Mode: mode,
Typeflag: tar.TypeDir,
},
}
}
func ociLayoutFile(version string) tarRecord {
if version == "" {
version = ocispec.ImageLayoutVersion
}
layout := ocispec.ImageLayout{
Version: version,
}
b, err := json.Marshal(layout)
if err != nil {
panic(err)
}
return tarRecord{
Header: &tar.Header{
Name: ocispec.ImageLayoutFile,
Mode: 0444,
Size: int64(len(b)),
Typeflag: tar.TypeReg,
},
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
n, err := w.Write(b)
return int64(n), err
},
}
}
func ociIndexRecord(manifests []ocispec.Descriptor) tarRecord {
index := ocispec.Index{
Versioned: ocispecs.Versioned{
SchemaVersion: 2,
},
Manifests: manifests,
}
b, err := json.Marshal(index)
if err != nil {
panic(err)
}
return tarRecord{
Header: &tar.Header{
Name: "index.json",
Mode: 0644,
Size: int64(len(b)),
Typeflag: tar.TypeReg,
},
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
n, err := w.Write(b)
return int64(n), err
},
}
}
type exportManifest struct {
manifest ocispec.Descriptor
names []string
}
func manifestsRecord(ctx context.Context, store content.Provider, manifests map[digest.Digest]*exportManifest) (tarRecord, error) {
mfsts := make([]struct {
Config string
RepoTags []string
Layers []string
}, len(manifests))
var i int
for _, m := range manifests {
p, err := content.ReadBlob(ctx, store, m.manifest)
if err != nil {
return tarRecord{}, err
}
var manifest ocispec.Manifest
if err := json.Unmarshal(p, &manifest); err != nil {
return tarRecord{}, err
}
if err := manifest.Config.Digest.Validate(); err != nil {
return tarRecord{}, fmt.Errorf("invalid manifest %q: %w", m.manifest.Digest, err)
}
dgst := manifest.Config.Digest
if err := dgst.Validate(); err != nil {
return tarRecord{}, err
}
mfsts[i].Config = path.Join("blobs", dgst.Algorithm().String(), dgst.Encoded())
for _, l := range manifest.Layers {
path := path.Join("blobs", l.Digest.Algorithm().String(), l.Digest.Encoded())
mfsts[i].Layers = append(mfsts[i].Layers, path)
}
for _, name := range m.names {
nname, err := familiarizeReference(name)
if err != nil {
return tarRecord{}, err
}
mfsts[i].RepoTags = append(mfsts[i].RepoTags, nname)
}
i++
}
b, err := json.Marshal(mfsts)
if err != nil {
return tarRecord{}, err
}
return tarRecord{
Header: &tar.Header{
Name: "manifest.json",
Mode: 0644,
Size: int64(len(b)),
Typeflag: tar.TypeReg,
},
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
n, err := w.Write(b)
return int64(n), err
},
}, nil
}
func writeTar(ctx context.Context, tw *tar.Writer, recordsWithEmpty []tarRecord) error {
var records []tarRecord
for _, r := range recordsWithEmpty {
if r.Header != nil {
records = append(records, r)
}
}
sort.Slice(records, func(i, j int) bool {
return records[i].Header.Name < records[j].Header.Name
})
var last string
for _, record := range records {
if record.Header.Name == last {
continue
}
last = record.Header.Name
if err := tw.WriteHeader(record.Header); err != nil {
return err
}
if record.CopyTo != nil {
n, err := record.CopyTo(ctx, tw)
if err != nil {
return err
}
if n != record.Header.Size {
return fmt.Errorf("unexpected copy size for %s", record.Header.Name)
}
} else if record.Header.Size > 0 {
return fmt.Errorf("no content to write to record with non-zero size for %s", record.Header.Name)
}
}
return nil
}