Merge pull request #46533 from vvoland/c8d-save-multiple-repo

c8d/save-load: Reimplement non-c8d idiosyncrasies

daemon/containerd/image.go

@@ -334,3 +334,10 @@ func (i *ImageService) resolveImage(ctx context.Context, refOrID string) (contai
 
 	return containerdimages.Image{}, images.ErrImageDoesNotExist{Ref: parsed}
 }
+
+// getAllImagesWithRepository returns a slice of images which name is a reference
+// pointing to the same repository as the given reference.
+func (i *ImageService) getAllImagesWithRepository(ctx context.Context, ref reference.Named) ([]containerdimages.Image, error) {
+	nameFilter := "^" + regexp.QuoteMeta(ref.Name()) + ":" + reference.TagRegexp.String() + "$"
+	return i.client.ImageService().List(ctx, "name~="+strconv.Quote(nameFilter))
+}

daemon/containerd/image_exporter.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/content"
@@ -16,6 +17,7 @@ import (
 	"github.com/distribution/reference"
 	"github.com/docker/docker/api/types/events"
 	"github.com/docker/docker/container"
+	"github.com/docker/docker/daemon/images"
 	"github.com/docker/docker/errdefs"
 	dockerarchive "github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/platforms"
@@ -78,13 +80,12 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea
 		}
 	}()
 
-	for _, name := range names {
+	addLease := func(ctx context.Context, target ocispec.Descriptor) error {
-		target, err := i.resolveDescriptor(ctx, name)
+		return leaseContent(ctx, contentStore, leasesManager, lease, target)
-		if err != nil {
+	}
-			return err
-		}
 
-		if err = leaseContent(ctx, contentStore, leasesManager, lease, target); err != nil {
+	exportImage := func(ctx context.Context, target ocispec.Descriptor, ref reference.Named) error {
+		if err := addLease(ctx, target); err != nil {
 			return err
 		}
 
@@ -99,15 +100,27 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea
 			target = desc
 		}
 
-		if ref, err := reference.ParseNormalizedNamed(name); err == nil {
+		if ref != nil {
-			ref = reference.TagNameOnly(ref)
 			opts = append(opts, archive.WithManifest(target, ref.String()))
 
 			log.G(ctx).WithFields(log.Fields{
 				"target": target,
-				"name":   ref.String(),
+				"name":   ref,
 			}).Debug("export image")
 		} else {
+			orgTarget := target
+			target.Annotations = make(map[string]string)
+
+			for k, v := range orgTarget.Annotations {
+				switch k {
+				case containerdimages.AnnotationImageName, ocispec.AnnotationRefName:
+					// Strip image name/tag annotations from the descriptor.
+					// Otherwise containerd will use it as name.
+				default:
+					target.Annotations[k] = v
+				}
+			}
+
 			opts = append(opts, archive.WithManifest(target))
 
 			log.G(ctx).WithFields(log.Fields{
@@ -116,6 +129,84 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea
 		}
 
 		i.LogImageEvent(target.Digest.String(), target.Digest.String(), events.ActionSave)
+		return nil
+	}
+
+	exportRepository := func(ctx context.Context, ref reference.Named) error {
+		imgs, err := i.getAllImagesWithRepository(ctx, ref)
+		if err != nil {
+			return errdefs.System(fmt.Errorf("failed to list all images from repository %s: %w", ref.Name(), err))
+		}
+
+		if len(imgs) == 0 {
+			return images.ErrImageDoesNotExist{Ref: ref}
+		}
+
+		for _, img := range imgs {
+			ref, err := reference.ParseNamed(img.Name)
+
+			if err != nil {
+				log.G(ctx).WithFields(log.Fields{
+					"image": img.Name,
+					"error": err,
+				}).Warn("couldn't parse image name as a valid named reference")
+				continue
+			}
+
+			if err := exportImage(ctx, img.Target, ref); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	for _, name := range names {
+		target, resolveErr := i.resolveDescriptor(ctx, name)
+
+		// Check if the requested name is a truncated digest of the resolved descriptor.
+		// If yes, that means that the user specified a specific image ID so
+		// it's not referencing a repository.
+		specificDigestResolved := false
+		if resolveErr == nil {
+			nameWithoutDigestAlgorithm := strings.TrimPrefix(name, target.Digest.Algorithm().String()+":")
+			specificDigestResolved = strings.HasPrefix(target.Digest.Encoded(), nameWithoutDigestAlgorithm)
+		}
+
+		log.G(ctx).WithFields(log.Fields{
+			"name":                   name,
+			"resolveErr":             resolveErr,
+			"specificDigestResolved": specificDigestResolved,
+		}).Debug("export requested")
+
+		ref, refErr := reference.ParseNormalizedNamed(name)
+
+		if resolveErr != nil || !specificDigestResolved {
+			// Name didn't resolve to anything, or name wasn't explicitly referencing a digest
+			if refErr == nil && reference.IsNameOnly(ref) {
+				// Reference is valid, but doesn't include a specific tag.
+				// Export all images with the same repository.
+				if err := exportRepository(ctx, ref); err != nil {
+					return err
+				}
+				continue
+			}
+		}
+
+		if resolveErr != nil {
+			return resolveErr
+		}
+		if refErr != nil {
+			return refErr
+		}
+
+		// If user exports a specific digest, it shouldn't have a tag.
+		if specificDigestResolved {
+			ref = nil
+		}
+		if err := exportImage(ctx, target, ref); err != nil {
+			return err
+		}
 	}
 
 	return i.client.Export(ctx, outStream, opts...)
@@ -187,7 +278,7 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt
 			name = img.Target.Digest.String()
 			loadedMsg = "Loaded image ID"
 		} else if named, err := reference.ParseNormalizedNamed(img.Name); err == nil {
-			name = reference.FamiliarName(reference.TagNameOnly(named))
+			name = reference.FamiliarString(reference.TagNameOnly(named))
 		}
 
 		err = i.walkImageManifests(ctx, img, func(platformImg *ImageManifest) error {

daemon/containerd/image_push.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"regexp"
-	"strconv"
 	"strings"
 	"sync"
 
@@ -50,8 +48,7 @@ func (i *ImageService) PushImage(ctx context.Context, sourceRef reference.Named,
 			// Image is not tagged nor digested, that means all tags push was requested.
 
 			// Find all images with the same repository.
-			nameFilter := "^" + regexp.QuoteMeta(sourceRef.Name()) + ":" + reference.TagRegexp.String() + "$"
+			imgs, err := i.getAllImagesWithRepository(ctx, sourceRef)
-			imgs, err := i.client.ImageService().List(ctx, "name~="+strconv.Quote(nameFilter))
 			if err != nil {
 				return err
 			}

integration-cli/docker_cli_save_load_unix_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/docker/docker/testutil"
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/icmd"
+	"gotest.tools/v3/skip"
 )
 
 // save a repo and try to load it using stdout
@@ -71,6 +72,9 @@ func (s *DockerCLISaveLoadSuite) TestSaveAndLoadRepoStdout(c *testing.T) {
 }
 
 func (s *DockerCLISaveLoadSuite) TestSaveAndLoadWithProgressBar(c *testing.T) {
+	// TODO(vvoland): https://github.com/moby/moby/issues/43910
+	skip.If(c, testEnv.UsingSnapshotter(), "TODO: Not implemented yet")
+
 	name := "test-load"
 	buildImageSuccessfully(c, name, build.WithDockerfile(`FROM busybox
 	RUN touch aa

integration/image/save_test.go

@@ -113,12 +113,16 @@ func TestSaveRepoWithMultipleImages(t *testing.T) {
 		return res.ID
 	}
 
+	busyboxImg, _, err := client.ImageInspectWithRaw(ctx, "busybox:latest")
+	assert.NilError(t, err)
+
 	repoName := "foobar-save-multi-images-test"
 	tagFoo := repoName + ":foo"
 	tagBar := repoName + ":bar"
 
 	idFoo := makeImage("busybox:latest", tagFoo)
 	idBar := makeImage("busybox:latest", tagBar)
+	idBusybox := busyboxImg.ID
 
 	client.ImageRemove(ctx, repoName, types.ImageRemoveOptions{Force: true})
 
@@ -142,20 +146,26 @@ func TestSaveRepoWithMultipleImages(t *testing.T) {
 		assert.Check(t, cmp.Nil(err))
 	}
 
-	// make the list of expected layers
+	expected := []string{idBusybox, idFoo, idBar}
-	img, _, err := client.ImageInspectWithRaw(ctx, "busybox:latest")
-	assert.NilError(t, err)
-
-	expected := []string{img.ID, idFoo, idBar}
-
 	// prefixes are not in tar
 	for i := range expected {
 		expected[i] = digest.Digest(expected[i]).Encoded()
 	}
 
-	sort.Strings(actual)
+	// With snapshotters, ID of the image is the ID of the manifest/index
-	sort.Strings(expected)
+	// With graphdrivers, ID of the image is the ID of the image config
-	assert.Assert(t, cmp.DeepEqual(actual, expected), "archive does not contains the right layers: got %v, expected %v", actual, expected)
+	if testEnv.UsingSnapshotter() {
+		// ID of image won't match the Config ID from manifest.json
+		// Just check if manifests exist in blobs
+		for _, blob := range expected {
+			_, err := fs.Stat(tarfs, "blobs/sha256/"+blob)
+			assert.Check(t, cmp.Nil(err))
+		}
+	} else {
+		sort.Strings(actual)
+		sort.Strings(expected)
+		assert.Assert(t, cmp.DeepEqual(actual, expected), "archive does not contains the right layers: got %v, expected %v", actual, expected)
+	}
 }
 
 func TestSaveDirectoryPermissions(t *testing.T) {