vendor: github.com/containerd/cgroups/v3 v3.0.3

release notes:

- NewSystemd handles UnitExists when starting units
- makefile fixes
- cgroups2: export memory max usage and swap max usage
- build(deps): bump github.com/cilium/ebpf from v0.9.1 to v0.11.0
- support psi
- feat: add Threads for cgroupv2
- Linux.Swap is defined as memory+swap combined, while in cgroup v2 swap is a separate value
- fix(): support re-enabling oom killer refs #307 by @kestrelcjx

full diff: https://github.com/containerd/cgroups/compare/v3.0.2...v3.0.3

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

vendor.mod

@@ -23,7 +23,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.15.17
 	github.com/aws/smithy-go v1.13.5
 	github.com/cloudflare/cfssl v1.6.4
-	github.com/containerd/cgroups/v3 v3.0.2
+	github.com/containerd/cgroups/v3 v3.0.3
 	github.com/containerd/containerd v1.7.11
 	github.com/containerd/continuity v0.4.2
 	github.com/containerd/fifo v1.1.0

vendor.sum

@@ -291,8 +291,8 @@ github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1
 github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
 github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
 github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
-github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0=
-github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE=
+github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0=
+github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
 github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=

vendor/github.com/containerd/cgroups/v3/Makefile

@@ -13,14 +13,17 @@
 #   limitations under the License.
 
 PACKAGES=$(shell go list ./... | grep -v /vendor/)
+GO_TAGS=$(if $(GO_BUILDTAGS),-tags "$(strip $(GO_BUILDTAGS))",)
+GO ?= go
+GO_BUILD_FLAGS ?=
 
 all: cgutil
-	go build -v
+	$(GO) build -v $(GO_TAGS)
 
 cgutil:
-	cd cmd/cgctl && go build -v
+	cd cmd/cgctl && $(GO) build $(GO_BUILD_FLAGS) -v $(GO_TAGS)
 
 proto:
 	protobuild --quiet ${PACKAGES}
 	# Keep them Go-idiomatic and backward-compatible with the gogo/protobuf era.
-	go-fix-acronym -w -a '(Cpu|Tcp|Rss)' $(shell find cgroup1/stats/ cgroup2/stats/ -name '*.pb.go')
+	go-fix-acronym -w -a '(Cpu|Tcp|Rss|Psi)' $(shell find cgroup1/stats/ cgroup2/stats/ -name '*.pb.go')

vendor/github.com/containerd/cgroups/v3/cgroup1/memory.go

@@ -454,6 +454,9 @@ func getOomControlValue(mem *specs.LinuxMemory) *int64 {
 	if mem.DisableOOMKiller != nil && *mem.DisableOOMKiller {
 		i := int64(1)
 		return &i
+	} else if mem.DisableOOMKiller != nil && !*mem.DisableOOMKiller {
+		i := int64(0)
+		return &i
 	}
 	return nil
 }

vendor/github.com/containerd/cgroups/v3/cgroup2/manager.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 	"math"
 	"os"
 	"path/filepath"
@@ -478,9 +479,9 @@ func (c *Manager) Delete() error {
 	return remove(c.path)
 }
 
-func (c *Manager) Procs(recursive bool) ([]uint64, error) {
-	var processes []uint64
-	err := filepath.Walk(c.path, func(p string, info os.FileInfo, err error) error {
+func (c *Manager) getTasks(recursive bool, tType string) ([]uint64, error) {
+	var tasks []uint64
+	err := filepath.Walk(c.path, func(p string, info fs.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -491,17 +492,25 @@ func (c *Manager) Procs(recursive bool) ([]uint64, error) {
 			return filepath.SkipDir
 		}
 		_, name := filepath.Split(p)
-		if name != cgroupProcs {
+		if name != tType {
 			return nil
 		}
-		procs, err := parseCgroupProcsFile(p)
+		curTasks, err := parseCgroupTasksFile(p)
 		if err != nil {
 			return err
 		}
-		processes = append(processes, procs...)
+		tasks = append(tasks, curTasks...)
 		return nil
 	})
-	return processes, err
+	return tasks, err
+}
+
+func (c *Manager) Procs(recursive bool) ([]uint64, error) {
+	return c.getTasks(recursive, cgroupProcs)
+}
+
+func (c *Manager) Threads(recursive bool) ([]uint64, error) {
+	return c.getTasks(recursive, cgroupThreads)
 }
 
 func (c *Manager) MoveTo(destination *Manager) error {
@@ -559,6 +568,7 @@ func (c *Manager) Stat() (*stats.Metrics, error) {
 		NrPeriods:     out["nr_periods"],
 		NrThrottled:   out["nr_throttled"],
 		ThrottledUsec: out["throttled_usec"],
+		PSI:           getStatPSIFromFile(filepath.Join(c.path, "cpu.pressure")),
 	}
 	metrics.Memory = &stats.MemoryStat{
 		Anon:                  out["anon"],
@@ -594,8 +604,11 @@ func (c *Manager) Stat() (*stats.Metrics, error) {
 		ThpCollapseAlloc:      out["thp_collapse_alloc"],
 		Usage:                 getStatFileContentUint64(filepath.Join(c.path, "memory.current")),
 		UsageLimit:            getStatFileContentUint64(filepath.Join(c.path, "memory.max")),
+		MaxUsage:              getStatFileContentUint64(filepath.Join(c.path, "memory.peak")),
 		SwapUsage:             getStatFileContentUint64(filepath.Join(c.path, "memory.swap.current")),
 		SwapLimit:             getStatFileContentUint64(filepath.Join(c.path, "memory.swap.max")),
+		SwapMaxUsage:          getStatFileContentUint64(filepath.Join(c.path, "memory.swap.peak")),
+		PSI:                   getStatPSIFromFile(filepath.Join(c.path, "memory.pressure")),
 	}
 	if len(memoryEvents) > 0 {
 		metrics.MemoryEvents = &stats.MemoryEvents{
@@ -606,7 +619,10 @@ func (c *Manager) Stat() (*stats.Metrics, error) {
 			OomKill: memoryEvents["oom_kill"],
 		}
 	}
-	metrics.Io = &stats.IOStat{Usage: readIoStats(c.path)}
+	metrics.Io = &stats.IOStat{
+		Usage: readIoStats(c.path),
+		PSI:   getStatPSIFromFile(filepath.Join(c.path, "io.pressure")),
+	}
 	metrics.Rdma = &stats.RdmaStat{
 		Current: rdmaStats(filepath.Join(c.path, "rdma.current")),
 		Limit:   rdmaStats(filepath.Join(c.path, "rdma.max")),
@@ -870,14 +886,7 @@ func NewSystemd(slice, group string, pid int, resources *Resources) (*Manager, e
 			newSystemdProperty("TasksMax", uint64(resources.Pids.Max)))
 	}
 
-	statusChan := make(chan string, 1)
-	if _, err := conn.StartTransientUnitContext(ctx, group, "replace", properties, statusChan); err == nil {
-		select {
-		case <-statusChan:
-		case <-time.After(time.Second):
-			logrus.Warnf("Timed out while waiting for StartTransientUnit(%s) completion signal from dbus. Continuing...", group)
-		}
-	} else if !isUnitExists(err) {
+	if err := startUnit(conn, group, properties, pid == -1); err != nil {
 		return &Manager{}, err
 	}
 
@@ -886,6 +895,60 @@ func NewSystemd(slice, group string, pid int, resources *Resources) (*Manager, e
 	}, nil
 }
 
+func startUnit(conn *systemdDbus.Conn, group string, properties []systemdDbus.Property, ignoreExists bool) error {
+	ctx := context.TODO()
+
+	statusChan := make(chan string, 1)
+	defer close(statusChan)
+
+	retry := true
+	started := false
+
+	for !started {
+		if _, err := conn.StartTransientUnitContext(ctx, group, "replace", properties, statusChan); err != nil {
+			if !isUnitExists(err) {
+				return err
+			}
+
+			if ignoreExists {
+				return nil
+			}
+
+			if retry {
+				retry = false
+				// When a unit of the same name already exists, it may be a leftover failed unit.
+				// If we reset it once, systemd can try to remove it.
+				attemptFailedUnitReset(conn, group)
+				continue
+			}
+
+			return err
+		} else {
+			started = true
+		}
+	}
+
+	select {
+	case s := <-statusChan:
+		if s != "done" {
+			attemptFailedUnitReset(conn, group)
+			return fmt.Errorf("error creating systemd unit `%s`: got `%s`", group, s)
+		}
+	case <-time.After(30 * time.Second):
+		logrus.Warnf("Timed out while waiting for StartTransientUnit(%s) completion signal from dbus. Continuing...", group)
+	}
+
+	return nil
+}
+
+func attemptFailedUnitReset(conn *systemdDbus.Conn, group string) {
+	err := conn.ResetFailedUnitContext(context.TODO(), group)
+
+	if err != nil {
+		logrus.Warnf("Unable to reset failed unit: %v", err)
+	}
+}
+
 func LoadSystemd(slice, group string) (*Manager, error) {
 	if slice == "" {
 		slice = defaultSlice

vendor/github.com/containerd/cgroups/v3/cgroup2/stats/metrics.pb.txt

@@ -60,6 +60,56 @@ file {
       json_name: "memoryEvents"
     }
   }
+  message_type {
+    name: "PSIData"
+    field {
+      name: "avg10"
+      number: 1
+      label: LABEL_OPTIONAL
+      type: TYPE_DOUBLE
+      json_name: "avg10"
+    }
+    field {
+      name: "avg60"
+      number: 2
+      label: LABEL_OPTIONAL
+      type: TYPE_DOUBLE
+      json_name: "avg60"
+    }
+    field {
+      name: "avg300"
+      number: 3
+      label: LABEL_OPTIONAL
+      type: TYPE_DOUBLE
+      json_name: "avg300"
+    }
+    field {
+      name: "total"
+      number: 4
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT64
+      json_name: "total"
+    }
+  }
+  message_type {
+    name: "PSIStats"
+    field {
+      name: "some"
+      number: 1
+      label: LABEL_OPTIONAL
+      type: TYPE_MESSAGE
+      type_name: ".io.containerd.cgroups.v2.PSIData"
+      json_name: "some"
+    }
+    field {
+      name: "full"
+      number: 2
+      label: LABEL_OPTIONAL
+      type: TYPE_MESSAGE
+      type_name: ".io.containerd.cgroups.v2.PSIData"
+      json_name: "full"
+    }
+  }
   message_type {
     name: "PidsStat"
     field {
@@ -121,6 +171,14 @@ file {
       type: TYPE_UINT64
       json_name: "throttledUsec"
     }
+    field {
+      name: "psi"
+      number: 7
+      label: LABEL_OPTIONAL
+      type: TYPE_MESSAGE
+      type_name: ".io.containerd.cgroups.v2.PSIStats"
+      json_name: "psi"
+    }
   }
   message_type {
     name: "MemoryStat"
@@ -369,6 +427,28 @@ file {
       type: TYPE_UINT64
       json_name: "swapLimit"
     }
+    field {
+      name: "max_usage"
+      number: 36
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT64
+      json_name: "maxUsage"
+    }
+    field {
+      name: "swap_max_usage"
+      number: 37
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT64
+      json_name: "swapMaxUsage"
+    }
+    field {
+      name: "psi"
+      number: 38
+      label: LABEL_OPTIONAL
+      type: TYPE_MESSAGE
+      type_name: ".io.containerd.cgroups.v2.PSIStats"
+      json_name: "psi"
+    }
   }
   message_type {
     name: "MemoryEvents"
@@ -461,6 +541,14 @@ file {
       type_name: ".io.containerd.cgroups.v2.IOEntry"
       json_name: "usage"
     }
+    field {
+      name: "psi"
+      number: 2
+      label: LABEL_OPTIONAL
+      type: TYPE_MESSAGE
+      type_name: ".io.containerd.cgroups.v2.PSIStats"
+      json_name: "psi"
+    }
   }
   message_type {
     name: "IOEntry"

vendor/github.com/containerd/cgroups/v3/cgroup2/stats/metrics.proto

@@ -14,6 +14,18 @@ message Metrics {
 	MemoryEvents memory_events = 8;
 }
 
+message PSIData {
+	double avg10 = 1;
+	double avg60 = 2;
+	double avg300 = 3;
+	uint64 total = 4;
+}
+
+message PSIStats {
+	PSIData some = 1;
+	PSIData full = 2;
+}
+
 message PidsStat {
 	uint64 current = 1;
 	uint64 limit = 2;
@@ -26,6 +38,7 @@ message CPUStat {
 	uint64 nr_periods = 4;
 	uint64 nr_throttled = 5;
 	uint64 throttled_usec = 6;
+	PSIStats psi = 7;
 }
 
 message MemoryStat {
@@ -64,6 +77,9 @@ message MemoryStat {
 	uint64 usage_limit = 33;
 	uint64 swap_usage = 34;
 	uint64 swap_limit = 35;
+	uint64 max_usage = 36;
+	uint64 swap_max_usage = 37;
+	PSIStats psi = 38;
 }
 
 message MemoryEvents {
@@ -87,6 +103,7 @@ message RdmaEntry {
 
 message IOStat {
 	repeated IOEntry usage = 1;
+	PSIStats psi = 2;
 }
 
 message IOEntry {

vendor/github.com/containerd/cgroups/v3/cgroup2/utils.go

@@ -68,8 +68,9 @@ func remove(path string) error {
 	return fmt.Errorf("cgroups: unable to remove path %q: %w", path, err)
 }
 
-// parseCgroupProcsFile parses /sys/fs/cgroup/$GROUPPATH/cgroup.procs
-func parseCgroupProcsFile(path string) ([]uint64, error) {
+// parseCgroupTasksFile parses /sys/fs/cgroup/$GROUPPATH/cgroup.procs or
+// /sys/fs/cgroup/$GROUPPATH/cgroup.threads
+func parseCgroupTasksFile(path string) ([]uint64, error) {
 	f, err := os.Open(path)
 	if err != nil {
 		return nil, err
@@ -176,6 +177,10 @@ func ToResources(spec *specs.LinuxResources) *Resources {
 		resources.Memory = &Memory{}
 		if swap := mem.Swap; swap != nil {
 			resources.Memory.Swap = swap
+			if l := mem.Limit; l != nil {
+				reduce := *swap - *l
+				resources.Memory.Swap = &reduce
+			}
 		}
 		if l := mem.Limit; l != nil {
 			resources.Memory.Max = l
@@ -479,6 +484,74 @@ func getHugePageSizeFromFilenames(fileNames []string) ([]string, error) {
 	return pageSizes, warn
 }
 
+func getStatPSIFromFile(path string) *stats.PSIStats {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil
+	}
+	defer f.Close()
+
+	psistats := &stats.PSIStats{}
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		parts := strings.Fields(sc.Text())
+		var pv *stats.PSIData
+		switch parts[0] {
+		case "some":
+			psistats.Some = &stats.PSIData{}
+			pv = psistats.Some
+		case "full":
+			psistats.Full = &stats.PSIData{}
+			pv = psistats.Full
+		}
+		if pv != nil {
+			err = parsePSIData(parts[1:], pv)
+			if err != nil {
+				logrus.Errorf("failed to read file %s: %v", path, err)
+				return nil
+			}
+		}
+	}
+
+	if err := sc.Err(); err != nil {
+		logrus.Errorf("unable to parse PSI data: %v", err)
+		return nil
+	}
+	return psistats
+}
+
+func parsePSIData(psi []string, data *stats.PSIData) error {
+	for _, f := range psi {
+		kv := strings.SplitN(f, "=", 2)
+		if len(kv) != 2 {
+			return fmt.Errorf("invalid PSI data: %q", f)
+		}
+		var pv *float64
+		switch kv[0] {
+		case "avg10":
+			pv = &data.Avg10
+		case "avg60":
+			pv = &data.Avg60
+		case "avg300":
+			pv = &data.Avg300
+		case "total":
+			v, err := strconv.ParseUint(kv[1], 10, 64)
+			if err != nil {
+				return fmt.Errorf("invalid %s PSI value: %w", kv[0], err)
+			}
+			data.Total = v
+		}
+		if pv != nil {
+			v, err := strconv.ParseFloat(kv[1], 64)
+			if err != nil {
+				return fmt.Errorf("invalid %s PSI value: %w", kv[0], err)
+			}
+			*pv = v
+		}
+	}
+	return nil
+}
+
 func getSubreaper() (int, error) {
 	var i uintptr
 	if err := unix.Prctl(unix.PR_GET_CHILD_SUBREAPER, uintptr(unsafe.Pointer(&i)), 0, 0, 0); err != nil {

vendor/modules.txt

@@ -226,7 +226,7 @@ github.com/container-storage-interface/spec/lib/go/csi
 # github.com/containerd/cgroups v1.1.0
 ## explicit; go 1.17
 github.com/containerd/cgroups/stats/v1
-# github.com/containerd/cgroups/v3 v3.0.2
+# github.com/containerd/cgroups/v3 v3.0.3
 ## explicit; go 1.18
 github.com/containerd/cgroups/v3
 github.com/containerd/cgroups/v3/cgroup1