diff --git a/daemon/cluster/executor/container/controller.go b/daemon/cluster/executor/container/controller.go
index b86aa4ecb3..32e03692d0 100644
--- a/daemon/cluster/executor/container/controller.go
+++ b/daemon/cluster/executor/container/controller.go
@@ -87,6 +87,11 @@ func (r *controller) Prepare(ctx context.Context) error {
 
 	if os.Getenv("DOCKER_SERVICE_PREFER_OFFLINE_IMAGE") != "1" {
 		if err := r.adapter.pullImage(ctx); err != nil {
+			cause := errors.Cause(err)
+			if cause == context.Canceled || cause == context.DeadlineExceeded {
+				return err
+			}
+
 			// NOTE(stevvooe): We always try to pull the image to make sure we have
 			// the most up to date version. This will return an error, but we only
 			// log it. If the image truly doesn't exist, the create below will
diff --git a/distribution/errors.go b/distribution/errors.go
index a9a17bd261..1e7630e380 100644
--- a/distribution/errors.go
+++ b/distribution/errors.go
@@ -40,7 +40,11 @@ type fallbackError struct {
 
 // Error renders the FallbackError as a string.
 func (f fallbackError) Error() string {
-	return f.err.Error()
+	return f.Cause().Error()
+}
+
+func (f fallbackError) Cause() error {
+	return f.err
 }
 
 // shouldV2Fallback returns true if this error is a reason to fall back to v1.