From 294fc9762e6650f8f8c59f1a210e6b37a02936f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Fri, 20 Oct 2023 11:58:26 +0200 Subject: [PATCH] volume: Don't decrement refcount below 0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With both rootless and live restore enabled, there's some race condition which causes the container to be `Unmount`ed before the refcount is restored. This makes sure we don't underflow the refcount (uint64) when decrementing it. The root cause of this race condition still needs to be investigated and fixed, but at least this unflakies the `TestLiveRestore`. Signed-off-by: Paweł Gronowski --- volume/local/local.go | 12 ++++++++++-- volume/mounts/mounts.go | 12 ++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/volume/local/local.go b/volume/local/local.go index 1b9f580ff2..6e96aeea41 100644 --- a/volume/local/local.go +++ b/volume/local/local.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "reflect" + "runtime/debug" "strings" "sync" @@ -334,8 +335,15 @@ func (v *localVolume) Unmount(id string) error { // ultimately there's nothing that can be done. If we don't decrement the count // this volume can never be removed until a daemon restart occurs. if v.needsMount() { - v.active.count-- - logger.WithField("active mounts", v.active).Debug("Decremented active mount count") + // TODO: Remove once the real bug is fixed: https://github.com/moby/moby/issues/46508 + if v.active.count > 0 { + v.active.count-- + logger.WithField("active mounts", v.active).Debug("Decremented active mount count") + } else { + logger.Error("An attempt to decrement a zero mount count") + logger.Error(string(debug.Stack())) + return nil + } } if v.active.count > 0 { diff --git a/volume/mounts/mounts.go b/volume/mounts/mounts.go index 50f445b6a1..7a518a046f 100644 --- a/volume/mounts/mounts.go +++ b/volume/mounts/mounts.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "runtime/debug" "syscall" "github.com/containerd/log" @@ -88,6 +89,15 @@ func (m *MountPoint) Cleanup(ctx context.Context) error { return nil } + logger := log.G(ctx).WithFields(log.Fields{"active": m.active, "id": m.ID}) + + // TODO: Remove once the real bug is fixed: https://github.com/moby/moby/issues/46508 + if m.active == 0 { + logger.Error("An attempt to decrement a zero mount count") + logger.Error(string(debug.Stack())) + return nil + } + for _, p := range m.safePaths { if !p.IsValid() { continue @@ -108,6 +118,8 @@ func (m *MountPoint) Cleanup(ctx context.Context) error { } m.active-- + logger.Debug("MountPoint.Cleanup Decrement active count") + if m.active == 0 { m.ID = "" }