c8d/prune: Remove gc.ref labels from configs of deleted images

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>

daemon/containerd/handlers.go

@@ -9,6 +9,19 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
+// walkPresentChildren is a simple wrapper for containerdimages.Walk with
+// presentChildrenHandler wrapping a simple handler that only operates on
+// walked Descriptor and doesn't return any errror.
+// This is only a convenient helper to reduce boilerplate.
+func (i *ImageService) walkPresentChildren(ctx context.Context, target ocispec.Descriptor, f func(context.Context, ocispec.Descriptor)) error {
+	store := i.client.ContentStore()
+	return containerdimages.Walk(ctx, presentChildrenHandler(store, containerdimages.HandlerFunc(
+		func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+			f(ctx, desc)
+			return nil, nil
+		})), target)
+}
+
 // presentChildrenHandler is a handler wrapper which traverses all children
 // descriptors that are present in the store and calls specified handler.
 func presentChildrenHandler(store content.Store, h containerdimages.HandlerFunc) containerdimages.HandlerFunc {

daemon/containerd/image_delete.go

@@ -6,6 +6,9 @@ import (
 	"github.com/containerd/containerd/images"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/sirupsen/logrus"
 )
 
 // ImageDelete deletes the image referenced by the given imageRef from this
@@ -56,11 +59,25 @@ func (i *ImageService) ImageDelete(ctx context.Context, imageRef string, force,
 		return nil, err
 	}
 
+	possiblyDeletedConfigs := map[digest.Digest]struct{}{}
+	if err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, d ocispec.Descriptor) {
+		if images.IsConfigType(d.MediaType) {
+			possiblyDeletedConfigs[d.Digest] = struct{}{}
+		}
+	}); err != nil {
+		return nil, err
+	}
+
 	err = i.client.ImageService().Delete(ctx, img.Name, images.SynchronousDelete())
 	if err != nil {
 		return nil, err
 	}
 
+	// Workaround for: https://github.com/moby/buildkit/issues/3797
+	if err := i.unleaseSnapshotsFromDeletedConfigs(context.Background(), possiblyDeletedConfigs); err != nil {
+		logrus.WithError(err).Warn("failed to unlease snapshots")
+	}
+
 	imgID := string(img.Target.Digest)
 	i.LogImageEvent(imgID, imgID, "untag")
 	i.LogImageEvent(imgID, imgID, "delete")

daemon/containerd/image_prune.go

@@ -10,6 +10,7 @@ import (
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/errdefs"
 	"github.com/hashicorp/go-multierror"
+	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -108,23 +109,37 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu
 
 	logrus.WithField("images", imagesToPrune).Debug("pruning")
 
+	possiblyDeletedConfigs := map[digest.Digest]struct{}{}
+
+	// Workaround for https://github.com/moby/buildkit/issues/3797
+	defer func() {
+		if err := i.unleaseSnapshotsFromDeletedConfigs(context.Background(), possiblyDeletedConfigs); err != nil {
+			errs = multierror.Append(errs, err)
+		}
+	}()
+
 	for _, img := range imagesToPrune {
 		blobs := []ocispec.Descriptor{}
 
-		err = containerdimages.Walk(ctx, presentChildrenHandler(store, containerdimages.HandlerFunc(
-			func(_ context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
-				blobs = append(blobs, desc)
-				return nil, nil
-			})),
-			img.Target)
-
+		err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) {
+			blobs = append(blobs, desc)
+			if containerdimages.IsConfigType(desc.MediaType) {
+				possiblyDeletedConfigs[desc.Digest] = struct{}{}
+			}
+		})
 		if err != nil {
 			errs = multierror.Append(errs, err)
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return &report, errs
+			}
 			continue
 		}
 		err = is.Delete(ctx, img.Name, containerdimages.SynchronousDelete())
 		if err != nil && !cerrdefs.IsNotFound(err) {
 			errs = multierror.Append(errs, err)
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return &report, errs
+			}
 			continue
 		}
 
@@ -148,5 +163,65 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu
 			}
 		}
 	}
+
 	return &report, errs
 }
+
+// unleaseSnapshotsFromDeletedConfigs removes gc.ref.snapshot content label from configs that are not
+// referenced by any of the existing images.
+// This is a temporary solution to the rootfs snapshot not being deleted when there's a buildkit history
+// item referencing an image config.
+func (i *ImageService) unleaseSnapshotsFromDeletedConfigs(ctx context.Context, possiblyDeletedConfigs map[digest.Digest]struct{}) error {
+	is := i.client.ImageService()
+	store := i.client.ContentStore()
+
+	all, err := is.List(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to list images during snapshot lease removal")
+	}
+
+	var errs error
+	for _, img := range all {
+		err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) {
+			if containerdimages.IsConfigType(desc.MediaType) {
+				delete(possiblyDeletedConfigs, desc.Digest)
+			}
+		})
+		if err != nil {
+			errs = multierror.Append(errs, err)
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return errs
+			}
+			continue
+		}
+	}
+
+	// At this point, all configs that are used by any image has been removed from the slice
+	for cfgDigest := range possiblyDeletedConfigs {
+		info, err := store.Info(ctx, cfgDigest)
+		if err != nil {
+			if cerrdefs.IsNotFound(err) {
+				logrus.WithField("config", cfgDigest).Debug("config already gone")
+			} else {
+				errs = multierror.Append(errs, err)
+				if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+					return errs
+				}
+			}
+			continue
+		}
+
+		label := "containerd.io/gc.ref.snapshot." + i.StorageDriver()
+
+		delete(info.Labels, label)
+		_, err = store.Update(ctx, info, "labels."+label)
+		if err != nil {
+			errs = multierror.Append(errs, errors.Wrapf(err, "failed to remove gc.ref.snapshot label from %s", cfgDigest))
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return errs
+			}
+		}
+	}
+
+	return errs
+}