diff --git a/daemon/graphdriver/windows/windows.go b/daemon/graphdriver/windows/windows.go
index 75a3d00e3f..29d53dc427 100644
--- a/daemon/graphdriver/windows/windows.go
+++ b/daemon/graphdriver/windows/windows.go
@@ -243,8 +243,18 @@ func (d *Driver) Remove(id string) error {
 	if err != nil {
 		return err
 	}
-	os.RemoveAll(filepath.Join(d.info.HomeDir, "sysfile-backups", rID)) // ok to fail
-	return hcsshim.DestroyLayer(d.info, rID)
+
+	layerPath := filepath.Join(d.info.HomeDir, rID)
+	tmpID := fmt.Sprintf("%s-removing", rID)
+	tmpLayerPath := filepath.Join(d.info.HomeDir, tmpID)
+	if err := os.Rename(layerPath, tmpLayerPath); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	if err := hcsshim.DestroyLayer(d.info, tmpID); err != nil {
+		logrus.Errorf("Failed to DestroyLayer %s: %s", id, err)
+	}
+
+	return nil
 }
 
 // Get returns the rootfs path for the id. This will mount the dir at its given path.