daemon/c8d: Implement save and load
This makes the `docker save` and `docker load` work with the containerd image store. The archive is both OCI and Docker compatible. Saved archive will only contain content which is available locally. In case the saved image is a multi-platform manifest list, the behavior depends on the local availability of the content. This is to be reconsidered when we have the `--platform` option in the CLI. - If all manifests and their contents, referenced by the manifest list are present, then the manifest-list is exported directly and the ID will be the same. - If only one platform manifest is present, only that manifest is exported (the image id will change and will be the id of platform-specific manifest, instead of the full manifest list). - If multiple, but not all, platform manifests are available, a new manifest list will be created which will be a subset of the original manifest list. Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>
This commit is contained in:
Paweł Gronowski
parent
9052e33a10
commit
af32603ae3
1 changed files with 176 additions and 24 deletions
|
|
@ -2,13 +2,20 @@ package containerd
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/containerd/containerd"
|
||||
cerrdefs "github.com/containerd/containerd/errdefs"
|
||||
containerdimages "github.com/containerd/containerd/images"
|
||||
"github.com/containerd/containerd/images/archive"
|
||||
"github.com/containerd/containerd/platforms"
|
||||
cplatforms "github.com/containerd/containerd/platforms"
|
||||
"github.com/docker/distribution/reference"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/docker/docker/errdefs"
|
||||
"github.com/docker/docker/pkg/platforms"
|
||||
"github.com/docker/docker/pkg/streamformatter"
|
||||
"github.com/opencontainers/image-spec/specs-go"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
|
|
@ -20,18 +27,64 @@ import (
|
|||
//
|
||||
// TODO(thaJeztah): produce JSON stream progress response and image events; see https://github.com/moby/moby/issues/43910
|
||||
func (i *ImageService) ExportImage(ctx context.Context, names []string, outStream io.Writer) error {
|
||||
platform := platforms.AllPlatformsWithPreference(cplatforms.Default())
|
||||
opts := []archive.ExportOpt{
|
||||
archive.WithPlatform(platforms.Ordered(platforms.DefaultSpec())),
|
||||
archive.WithSkipNonDistributableBlobs(),
|
||||
|
||||
// This makes the exported archive also include `manifest.json`
|
||||
// when the image is a manifest list. It is needed for backwards
|
||||
// compatibility with Docker image format.
|
||||
// The containerd will choose only one manifest for the `manifest.json`.
|
||||
// Our preference is to have it point to the default platform.
|
||||
// Example:
|
||||
// Daemon is running on linux/arm64
|
||||
// When we export linux/amd64 and linux/arm64, manifest.json will point to linux/arm64.
|
||||
// When we export linux/amd64 only, manifest.json will point to linux/amd64.
|
||||
// Note: This is only applicable if importing this archive into non-containerd Docker.
|
||||
// Importing the same archive into containerd, will not restrict the platforms.
|
||||
archive.WithPlatform(platform),
|
||||
}
|
||||
is := i.client.ImageService()
|
||||
for _, imageRef := range names {
|
||||
named, err := reference.ParseDockerRef(imageRef)
|
||||
|
||||
ctx, release, err := i.client.WithLease(ctx)
|
||||
if err != nil {
|
||||
return errdefs.System(err)
|
||||
}
|
||||
defer release(ctx)
|
||||
|
||||
for _, name := range names {
|
||||
target, err := i.resolveDescriptor(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts = append(opts, archive.WithImage(is, named.String()))
|
||||
|
||||
// We may not have locally all the platforms that are specified in the index.
|
||||
// Export only those manifests that we have.
|
||||
// TODO(vvoland): Reconsider this when `--platform` is added.
|
||||
if containerdimages.IsIndexType(target.MediaType) {
|
||||
desc, err := i.getBestDescriptorForExport(ctx, target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target = desc
|
||||
}
|
||||
|
||||
if ref, err := reference.ParseNormalizedNamed(name); err == nil {
|
||||
ref = reference.TagNameOnly(ref)
|
||||
opts = append(opts, archive.WithManifest(target, ref.String()))
|
||||
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"target": target,
|
||||
"name": ref.String(),
|
||||
}).Debug("export image")
|
||||
} else {
|
||||
opts = append(opts, archive.WithManifest(target))
|
||||
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"target": target,
|
||||
}).Debug("export image without name")
|
||||
}
|
||||
}
|
||||
|
||||
return i.client.Export(ctx, outStream, opts...)
|
||||
}
|
||||
|
||||
|
|
@ -41,33 +94,132 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea
|
|||
//
|
||||
// TODO(thaJeztah): produce JSON stream progress response and image events; see https://github.com/moby/moby/issues/43910
|
||||
func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outStream io.Writer, quiet bool) error {
|
||||
platform := platforms.All
|
||||
// TODO(vvoland): Allow user to pass platform
|
||||
platform := cplatforms.All
|
||||
imgs, err := i.client.Import(ctx, inTar, containerd.WithImportPlatform(platform))
|
||||
|
||||
if err != nil {
|
||||
// TODO(thaJeztah): remove this log or change to debug once we can; see https://github.com/moby/moby/pull/43822#discussion_r937502405
|
||||
logrus.WithError(err).Warn("failed to import image to containerd")
|
||||
return errors.Wrap(err, "failed to import image")
|
||||
logrus.WithError(err).Debug("failed to import image to containerd")
|
||||
return errdefs.System(err)
|
||||
}
|
||||
|
||||
store := i.client.ContentStore()
|
||||
progress := streamformatter.NewStdoutWriter(outStream)
|
||||
|
||||
for _, img := range imgs {
|
||||
platformImg := containerd.NewImageWithPlatform(i.client, img, platform)
|
||||
|
||||
unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter)
|
||||
allPlatforms, err := containerdimages.Platforms(ctx, store, img.Target)
|
||||
if err != nil {
|
||||
// TODO(thaJeztah): remove this log or change to debug once we can; see https://github.com/moby/moby/pull/43822#discussion_r937502405
|
||||
logrus.WithError(err).WithField("image", img.Name).Debug("failed to check if image is unpacked")
|
||||
continue
|
||||
logrus.WithError(err).WithField("image", img.Name).Debug("failed to get image platforms")
|
||||
return errdefs.Unknown(err)
|
||||
}
|
||||
|
||||
if !unpacked {
|
||||
err := platformImg.Unpack(ctx, i.snapshotter)
|
||||
if err != nil {
|
||||
// TODO(thaJeztah): remove this log or change to debug once we can; see https://github.com/moby/moby/pull/43822#discussion_r937502405
|
||||
logrus.WithError(err).WithField("image", img.Name).Warn("failed to unpack image")
|
||||
return errors.Wrap(err, "failed to unpack image")
|
||||
}
|
||||
name := img.Name
|
||||
if named, err := reference.ParseNormalizedNamed(img.Name); err == nil {
|
||||
name = reference.FamiliarName(named)
|
||||
}
|
||||
|
||||
for _, platform := range allPlatforms {
|
||||
logger := logrus.WithFields(logrus.Fields{
|
||||
"platform": platform,
|
||||
"image": name,
|
||||
})
|
||||
platformImg := containerd.NewImageWithPlatform(i.client, img, cplatforms.OnlyStrict(platform))
|
||||
|
||||
unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter)
|
||||
if err != nil {
|
||||
logger.WithError(err).Debug("failed to check if image is unpacked")
|
||||
continue
|
||||
}
|
||||
|
||||
if !unpacked {
|
||||
err = platformImg.Unpack(ctx, i.snapshotter)
|
||||
|
||||
if err != nil {
|
||||
return errdefs.System(err)
|
||||
}
|
||||
}
|
||||
logger.WithField("alreadyUnpacked", unpacked).WithError(err).Debug("unpack")
|
||||
}
|
||||
|
||||
fmt.Fprintf(progress, "Loaded image: %s\n", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getBestDescriptorForExport returns a descriptor which only references content available locally.
|
||||
// The returned descriptor can be:
|
||||
// - The same index descriptor - if all content is available
|
||||
// - Platform specific manifest - if only one manifest from the whole index is available
|
||||
// - Reduced index descriptor - if not all, but more than one manifest is available
|
||||
//
|
||||
// The reduced index descriptor is stored in the content store and may be garbage collected.
|
||||
// It's advised to pass a context with a lease that's long enough to cover usage of the blob.
|
||||
func (i *ImageService) getBestDescriptorForExport(ctx context.Context, indexDesc ocispec.Descriptor) (ocispec.Descriptor, error) {
|
||||
none := ocispec.Descriptor{}
|
||||
|
||||
if !containerdimages.IsIndexType(indexDesc.MediaType) {
|
||||
err := fmt.Errorf("index/manifest-list descriptor expected, got: %s", indexDesc.MediaType)
|
||||
return none, errdefs.InvalidParameter(err)
|
||||
}
|
||||
store := i.client.ContentStore()
|
||||
children, err := containerdimages.Children(ctx, store, indexDesc)
|
||||
if err != nil {
|
||||
if cerrdefs.IsNotFound(err) {
|
||||
return none, errdefs.NotFound(err)
|
||||
}
|
||||
return none, errdefs.System(err)
|
||||
}
|
||||
|
||||
// Check which platform manifests have all their blobs available.
|
||||
hasMissingManifests := false
|
||||
var presentManifests []ocispec.Descriptor
|
||||
for _, mfst := range children {
|
||||
if containerdimages.IsManifestType(mfst.MediaType) {
|
||||
available, _, _, missing, err := containerdimages.Check(ctx, store, mfst, nil)
|
||||
if err != nil {
|
||||
hasMissingManifests = true
|
||||
logrus.WithField("manifest", mfst.Digest).Warn("failed to check manifest's blob availability, won't export")
|
||||
continue
|
||||
}
|
||||
|
||||
if available && len(missing) == 0 {
|
||||
presentManifests = append(presentManifests, mfst)
|
||||
logrus.WithField("manifest", mfst.Digest).Debug("manifest content present, will export")
|
||||
} else {
|
||||
hasMissingManifests = true
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"manifest": mfst.Digest,
|
||||
"missing": missing,
|
||||
}).Debug("manifest is missing, won't export")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !hasMissingManifests || len(children) == 0 {
|
||||
// If we have the full image, or it has no manifests, just export the original index.
|
||||
return indexDesc, nil
|
||||
} else if len(presentManifests) == 1 {
|
||||
// If only one platform is present, export that one manifest.
|
||||
return presentManifests[0], nil
|
||||
} else if len(presentManifests) == 0 {
|
||||
// Return error when none of the image's manifest is present.
|
||||
return none, errdefs.NotFound(fmt.Errorf("none of the manifests is fully present in the content store"))
|
||||
}
|
||||
|
||||
// Create a new index which contains only the manifests we have in store.
|
||||
index := ocispec.Index{
|
||||
Versioned: specs.Versioned{
|
||||
SchemaVersion: 2,
|
||||
},
|
||||
MediaType: ocispec.MediaTypeImageIndex,
|
||||
Manifests: presentManifests,
|
||||
Annotations: indexDesc.Annotations,
|
||||
}
|
||||
|
||||
reducedIndexDesc, err := storeJson(ctx, store, index.MediaType, index, nil)
|
||||
if err != nil {
|
||||
return none, err
|
||||
}
|
||||
|
||||
return reducedIndexDesc, nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue