diff --git a/daemon/containerd/cache.go b/daemon/containerd/cache.go
index 5f1621acbb..6133a0b89e 100644
--- a/daemon/containerd/cache.go
+++ b/daemon/containerd/cache.go
@@ -5,7 +5,6 @@ import (
 	"reflect"
 	"strings"
 
-	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	imagetype "github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/builder"
@@ -57,17 +56,11 @@ func (ic *localCache) GetCache(parentID string, cfg *container.Config) (imageID
 
 	// FROM scratch
 	if parentID == "" {
-		imgs, err := ic.imageService.Images(ctx, types.ImageListOptions{
-			All: true,
-		})
+		c, err := ic.imageService.getImagesWithLabel(ctx, imageLabelClassicBuilderFromScratch, "1")
 		if err != nil {
 			return "", err
 		}
-		for _, img := range imgs {
-			if img.ParentID == parentID {
-				children = append(children, image.ID(img.ID))
-			}
-		}
+		children = c
 	} else {
 		c, err := ic.imageService.Children(ctx, image.ID(parentID))
 		if err != nil {
diff --git a/daemon/containerd/image_builder.go b/daemon/containerd/image_builder.go
index 521211024c..ff985ea626 100644
--- a/daemon/containerd/image_builder.go
+++ b/daemon/containerd/image_builder.go
@@ -41,8 +41,13 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-// Digest of the image which was the base image of the committed container.
-const imageLabelClassicBuilderParent = "org.mobyproject.image.parent"
+const (
+	// Digest of the image which was the base image of the committed container.
+	imageLabelClassicBuilderParent = "org.mobyproject.image.parent"
+
+	// "1" means that the image was created directly from the "FROM scratch".
+	imageLabelClassicBuilderFromScratch = "org.mobyproject.image.fromscratch"
+)
 
 // GetImageAndReleasableLayer returns an image and releaseable layer for a
 // reference or ID. Every call to GetImageAndReleasableLayer MUST call
@@ -483,6 +488,10 @@ func (i *ImageService) createImageOCI(ctx context.Context, imgToCreate imagespec
 		},
 	}
 
+	if parentDigest == "" {
+		img.Labels[imageLabelClassicBuilderFromScratch] = "1"
+	}
+
 	createdImage, err := i.client.ImageService().Update(ctx, img)
 	if err != nil {
 		if !cerrdefs.IsNotFound(err) {
diff --git a/daemon/containerd/image_children.go b/daemon/containerd/image_children.go
index 38c95be096..414f05d5d7 100644
--- a/daemon/containerd/image_children.go
+++ b/daemon/containerd/image_children.go
@@ -10,9 +10,10 @@ import (
 	"github.com/pkg/errors"
 )
 
-// Children returns a slice of image IDs that are children of the `id` image
-func (i *ImageService) Children(ctx context.Context, id image.ID) ([]image.ID, error) {
-	imgs, err := i.images.List(ctx, "labels."+imageLabelClassicBuilderParent+"=="+string(id))
+// getImagesWithLabel returns all images that have the matching label key and value.
+func (i *ImageService) getImagesWithLabel(ctx context.Context, labelKey string, labelValue string) ([]image.ID, error) {
+	imgs, err := i.images.List(ctx, "labels."+labelKey+"=="+labelValue)
+
 	if err != nil {
 		return []image.ID{}, errdefs.System(errors.Wrap(err, "failed to list all images"))
 	}
@@ -25,6 +26,11 @@ func (i *ImageService) Children(ctx context.Context, id image.ID) ([]image.ID, e
 	return children, nil
 }
 
+// Children returns a slice of image IDs that are children of the `id` image
+func (i *ImageService) Children(ctx context.Context, id image.ID) ([]image.ID, error) {
+	return i.getImagesWithLabel(ctx, imageLabelClassicBuilderParent, string(id))
+}
+
 // parents returns a slice of image IDs that are parents of the `id` image
 //
 // Called from image_delete.go to prune dangling parents.