diff --git a/daemon/delete.go b/daemon/delete.go
index b5a85c727d..54ca5729bb 100644
--- a/daemon/delete.go
+++ b/daemon/delete.go
@@ -96,8 +96,8 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, config ty
 			err := fmt.Errorf("You cannot remove a %s container %s. %s", state, container.ID, procedure)
 			return errdefs.Conflict(err)
 		}
-		if err := daemon.Kill(container); err != nil {
-			return fmt.Errorf("Could not kill running container %s, cannot remove - %v", container.ID, err)
+		if err := daemon.Kill(container); err != nil && !isNotRunning(err) {
+			return fmt.Errorf("cannot remove container %q: could not kill: %w", container.Name, err)
 		}
 	}
 
diff --git a/daemon/errors.go b/daemon/errors.go
index 803c070f12..f0790cce87 100644
--- a/daemon/errors.go
+++ b/daemon/errors.go
@@ -10,10 +10,21 @@ import (
 	"google.golang.org/grpc/status"
 )
 
-func errNotRunning(id string) error {
-	return errdefs.Conflict(errors.Errorf("Container %s is not running", id))
+func isNotRunning(err error) bool {
+	var nre *containerNotRunningError
+	return errors.As(err, &nre)
 }
 
+func errNotRunning(id string) error {
+	return &containerNotRunningError{errors.Errorf("container %s is not running", id)}
+}
+
+type containerNotRunningError struct {
+	error
+}
+
+func (e containerNotRunningError) Conflict() {}
+
 func containerNotFound(id string) error {
 	return objNotFoundError{"container", id}
 }
diff --git a/daemon/errors_test.go b/daemon/errors_test.go
new file mode 100644
index 0000000000..1c79aa3361
--- /dev/null
+++ b/daemon/errors_test.go
@@ -0,0 +1,12 @@
+package daemon
+
+import (
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+func TestContainerNotRunningError(t *testing.T) {
+	err := errNotRunning("12345")
+	assert.Check(t, isNotRunning(err))
+}