diff --git a/image/cache/cache.go b/image/cache/cache.go
index 6d3f4c57b5..77cb9d1706 100644
--- a/image/cache/cache.go
+++ b/image/cache/cache.go
@@ -1,11 +1,13 @@
 package cache // import "github.com/docker/docker/image/cache"
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"reflect"
 	"strings"
 
+	"github.com/containerd/log"
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/dockerversion"
 	"github.com/docker/docker/image"
@@ -216,6 +218,22 @@ func getImageIDAndError(img *image.Image, err error) (string, error) {
 // created. nil is returned if a child cannot be found. An error is
 // returned if the parent image cannot be found.
 func getLocalCachedImage(imageStore image.Store, imgID image.ID, config *containertypes.Config) (*image.Image, error) {
+	if config == nil {
+		return nil, nil
+	}
+
+	isBuiltLocally := func(id image.ID) bool {
+		builtLocally, err := imageStore.IsBuiltLocally(id)
+		if err != nil {
+			log.G(context.TODO()).WithFields(log.Fields{
+				"error": err,
+				"id":    id,
+			}).Warn("failed to check if image was built locally")
+			return false
+		}
+		return builtLocally
+	}
+
 	// Loop on the children of the given image and check the config
 	getMatch := func(siblings []image.ID) (*image.Image, error) {
 		var match *image.Image
@@ -225,6 +243,10 @@ func getLocalCachedImage(imageStore image.Store, imgID image.ID, config *contain
 				return nil, fmt.Errorf("unable to find image %q", id)
 			}
 
+			if !isBuiltLocally(id) {
+				continue
+			}
+
 			if compare(&img.ContainerConfig, config) {
 				// check for the most up to date match
 				if match == nil || match.Created.Before(img.Created) {
@@ -238,11 +260,29 @@ func getLocalCachedImage(imageStore image.Store, imgID image.ID, config *contain
 	// In this case, this is `FROM scratch`, which isn't an actual image.
 	if imgID == "" {
 		images := imageStore.Map()
+
 		var siblings []image.ID
 		for id, img := range images {
-			if img.Parent == imgID {
-				siblings = append(siblings, id)
+			if img.Parent != "" {
+				continue
 			}
+
+			if !isBuiltLocally(id) {
+				continue
+			}
+
+			// Do a quick initial filter on the Cmd to avoid adding all
+			// non-local images with empty parent to the siblings slice and
+			// performing a full config compare.
+			//
+			// config.Cmd is set to the current Dockerfile instruction so we
+			// check it against the img.ContainerConfig.Cmd which is the
+			// command of the last layer.
+			if !strSliceEqual(img.ContainerConfig.Cmd, config.Cmd) {
+				continue
+			}
+
+			siblings = append(siblings, id)
 		}
 		return getMatch(siblings)
 	}
@@ -251,3 +291,15 @@ func getLocalCachedImage(imageStore image.Store, imgID image.ID, config *contain
 	siblings := imageStore.Children(imgID)
 	return getMatch(siblings)
 }
+
+func strSliceEqual(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := 0; i < len(a); i++ {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}