Add `--cpus` flag to control cpu resources

This fix tries to address the proposal raised in 27921 and add
`--cpus` flag for `docker run/create`.

Basically, `--cpus` will allow user to specify a number (possibly partial)
about how many CPUs the container will use. For example, on a 2-CPU system
`--cpus 1.5` means the container will take 75% (1.5/2) of the CPU share.

This fix adds a `NanoCPUs` field to `HostConfig` since swarmkit alreay
have a concept of NanoCPUs for tasks. The `--cpus` flag will translate
the number into reused `NanoCPUs` to be consistent.

This fix adds integration tests to cover the changes.

Related docs (`docker run` and Remote APIs) have been updated.

This fix fixes 27921.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

api/types/container/host_config.go

@@ -234,6 +234,7 @@ type Resources struct {
 	// Applicable to all platforms
 	CPUShares int64 `json:"CpuShares"` // CPU shares (relative weight vs. other containers)
 	Memory    int64 // Memory limit (in bytes)
+	NanoCPUs  int64 `json:"NanoCpus"` // CPU quota in units of 10<sup>-9</sup> CPUs.
 
 	// Applicable to UNIX platforms
 	CgroupParent         string // Parent cgroup.

cli/command/service/opts.go

@@ -2,7 +2,6 @@ package service
 
 import (
 	"fmt"
-	"math/big"
 	"strconv"
 	"strings"
 	"time"
@@ -40,33 +39,6 @@ func (m *memBytes) Value() int64 {
 	return int64(*m)
 }
 
-type nanoCPUs int64
-
-func (c *nanoCPUs) String() string {
-	return big.NewRat(c.Value(), 1e9).FloatString(3)
-}
-
-func (c *nanoCPUs) Set(value string) error {
-	cpu, ok := new(big.Rat).SetString(value)
-	if !ok {
-		return fmt.Errorf("Failed to parse %v as a rational number", value)
-	}
-	nano := cpu.Mul(cpu, big.NewRat(1e9, 1))
-	if !nano.IsInt() {
-		return fmt.Errorf("value is too precise")
-	}
-	*c = nanoCPUs(nano.Num().Int64())
-	return nil
-}
-
-func (c *nanoCPUs) Type() string {
-	return "NanoCPUs"
-}
-
-func (c *nanoCPUs) Value() int64 {
-	return int64(*c)
-}
-
 // PositiveDurationOpt is an option type for time.Duration that uses a pointer.
 // It bahave similarly to DurationOpt but only allows positive duration values.
 type PositiveDurationOpt struct {
@@ -156,9 +128,9 @@ type updateOptions struct {
 }
 
 type resourceOptions struct {
-	limitCPU      nanoCPUs
+	limitCPU      opts.NanoCPUs
 	limitMemBytes memBytes
-	resCPU        nanoCPUs
+	resCPU        opts.NanoCPUs
 	resMemBytes   memBytes
 }
 

cli/command/service/opts_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/opts"
 	"github.com/docker/docker/pkg/testutil/assert"
 )
 
@@ -21,12 +22,12 @@ func TestMemBytesSetAndValue(t *testing.T) {
 }
 
 func TestNanoCPUsString(t *testing.T) {
-	var cpus nanoCPUs = 6100000000
+	var cpus opts.NanoCPUs = 6100000000
 	assert.Equal(t, cpus.String(), "6.100")
 }
 
 func TestNanoCPUsSetAndValue(t *testing.T) {
-	var cpus nanoCPUs
+	var cpus opts.NanoCPUs
 	assert.NilError(t, cpus.Set("0.35"))
 	assert.Equal(t, cpus.Value(), int64(350000000))
 }

daemon/daemon_unix.go

@@ -15,6 +15,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/api/types"
@@ -110,6 +111,16 @@ func getCPUResources(config containertypes.Resources) *specs.CPU {
 		cpu.Mems = &cpuset
 	}
 
+	if config.NanoCPUs > 0 {
+		// Use the default setting of 100ms, as is specified in:
+		// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
+		//    	cpu.cfs_period_us=100ms
+		period := uint64(100 * time.Millisecond / time.Microsecond)
+		quota := uint64(config.NanoCPUs) * period / 1e9
+		cpu.Period = &period
+		cpu.Quota = &quota
+	}
+
 	if config.CPUPeriod != 0 {
 		period := uint64(config.CPUPeriod)
 		cpu.Period = &period
@@ -341,6 +352,19 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi
 	}
 
 	// cpu subsystem checks and adjustments
+	if resources.NanoCPUs > 0 && resources.CPUPeriod > 0 {
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Period cannot both be set")
+	}
+	if resources.NanoCPUs > 0 && resources.CPUQuota > 0 {
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Quota cannot both be set")
+	}
+	if resources.NanoCPUs > 0 && (!sysInfo.CPUCfsPeriod || !sysInfo.CPUCfsQuota) {
+		return warnings, fmt.Errorf("NanoCPUs can not be set, as your kernel does not support CPU cfs period/quota or the cgroup is not mounted")
+	}
+	if resources.NanoCPUs < 0 || resources.NanoCPUs > int64(sysinfo.NumCPU())*1e9 {
+		return warnings, fmt.Errorf("Range of Nano CPUs is from 1 to %d", int64(sysinfo.NumCPU())*1e9)
+	}
+
 	if resources.CPUShares > 0 && !sysInfo.CPUShares {
 		warnings = append(warnings, "Your kernel does not support CPU shares or the cgroup is not mounted. Shares discarded.")
 		logrus.Warn("Your kernel does not support CPU shares or the cgroup is not mounted. Shares discarded.")

daemon/daemon_windows.go

@@ -103,6 +103,17 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi
 		return warnings, fmt.Errorf("Conflicting options: CPU Shares and CPU Percent cannot both be set")
 	}
 
+	if resources.NanoCPUs > 0 && resources.CPUPercent > 0 {
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Percent cannot both be set")
+	}
+
+	if resources.NanoCPUs > 0 && resources.CPUShares > 0 {
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Shares cannot both be set")
+	}
+	if resources.NanoCPUs < 0 || resources.NanoCPUs > int64(sysinfo.NumCPU())*1e9 {
+		return warnings, fmt.Errorf("Range of Nano CPUs is from 1 to %d", int64(sysinfo.NumCPU())*1e9)
+	}
+
 	// TODO Windows: Add more validation of resource settings not supported on Windows
 
 	if resources.BlkioWeight > 0 {

daemon/oci_windows.go

@@ -6,6 +6,7 @@ import (
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/oci"
+	"github.com/docker/docker/pkg/sysinfo"
 	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
@@ -82,6 +83,9 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
 	// @darrenstahlmsft implement these resources
 	cpuShares := uint16(c.HostConfig.CPUShares)
 	cpuPercent := uint8(c.HostConfig.CPUPercent)
+	if c.HostConfig.NanoCPUs > 0 {
+		cpuPercent = uint8(c.HostConfig.NanoCPUs * 100 / int64(sysinfo.NumCPU()) / 1e9)
+	}
 	memoryLimit := uint64(c.HostConfig.Memory)
 	s.Windows.Resources = &specs.WindowsResources{
 		CPU: &specs.WindowsCPUResources{

docs/reference/api/docker_remote_api.md

@@ -164,6 +164,7 @@ This section lists each version from latest to oldest.  Each listing includes a
 * The `hostConfig` option now accepts the fields `CpuRealtimePeriod` and `CpuRtRuntime` to allocate cpu runtime to rt tasks when `CONFIG_RT_GROUP_SCHED` is enabled in the kernel.
 * The `SecurityOptions` field within the `GET /info` response now includes `userns` if user namespaces are enabled in the daemon.
 * `GET /nodes` and `GET /node/(id or name)` now return `Addr` as part of a node's `Status`, which is the address that that node connects to the manager from.
+* The `HostConfig` field now includes `NanoCPUs` that represents CPU quota in units of 10<sup>-9</sup> CPUs.
 
 ### v1.24 API changes
 

docs/reference/api/docker_remote_api_v1.25.md

@@ -302,6 +302,7 @@ Create a container
              "MemorySwap": 0,
              "MemoryReservation": 0,
              "KernelMemory": 0,
+             "NanoCPUs": 500000,
              "CpuPercent": 80,
              "CpuShares": 512,
              "CpuPeriod": 100000,
@@ -425,6 +426,7 @@ Create a container
           You must use this with `memory` and make the swap value larger than `memory`.
     -   **MemoryReservation** - Memory soft limit in bytes.
     -   **KernelMemory** - Kernel memory limit in bytes.
+    -   **NanoCPUs** - CPU quota in units of 10<sup>-9</sup> CPUs.
     -   **CpuPercent** - An integer value containing the usable percentage of the available CPUs. (Windows daemon only)
     -   **CpuShares** - An integer value containing the container's CPU Shares
           (ie. the relative weight vs other containers).

docs/reference/commandline/create.md

@@ -35,6 +35,7 @@ Options:
       --cpu-period int              Limit CPU CFS (Completely Fair Scheduler) period
       --cpu-quota int               Limit CPU CFS (Completely Fair Scheduler) quota
   -c, --cpu-shares int              CPU shares (relative weight)
+      --cpus NanoCPUs               Number of CPUs (default 0.000)
       --cpu-rt-period int           Limit the CPU real-time period in microseconds
       --cpu-rt-runtime int          Limit the CPU real-time runtime in microseconds
       --cpuset-cpus string          CPUs in which to allow execution (0-3, 0,1)

docs/reference/commandline/run.md

@@ -33,6 +33,7 @@ Options:
       --cpu-period int              Limit CPU CFS (Completely Fair Scheduler) period
       --cpu-quota int               Limit CPU CFS (Completely Fair Scheduler) quota
   -c, --cpu-shares int              CPU shares (relative weight)
+      --cpus NanoCPUs               Number of CPUs (default 0.000)
       --cpu-rt-period int           Limit the CPU real-time period in microseconds
       --cpu-rt-runtime int          Limit the CPU real-time runtime in microseconds
       --cpuset-cpus string          CPUs in which to allow execution (0-3, 0,1)

docs/reference/run.md

@@ -686,6 +686,7 @@ container:
 | `--memory-reservation=""`  | Memory soft limit (format: `<number>[<unit>]`). Number is a positive integer. Unit can be one of `b`, `k`, `m`, or `g`.                         |
 | `--kernel-memory=""`       | Kernel memory limit (format: `<number>[<unit>]`). Number is a positive integer. Unit can be one of `b`, `k`, `m`, or `g`. Minimum is 4M.        |
 | `-c`, `--cpu-shares=0`     | CPU shares (relative weight)                                                                                                                    |
+| `--cpus=0.000`             | Number of CPUs. Number is a fractional number. 0.000 means no limit.                                                                            |
 | `--cpu-period=0`           | Limit the CPU CFS (Completely Fair Scheduler) period                                                                                            |
 | `--cpuset-cpus=""`         | CPUs in which to allow execution (0-3, 0,1)                                                                                                     |
 | `--cpuset-mems=""`         | Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems.                                                     |
@@ -970,6 +971,13 @@ Examples:
 
 If there is 1 CPU, this means the container can get 50% CPU worth of run-time every 50ms.
 
+In addition to use `--cpu-period` and `--cpu-quota` for setting CPU period constraints,
+it is possible to specify `--cpus` with a float number to achieve the same purpose.
+For example, if there is 1 CPU, then `--cpus=0.5` will achieve the same result as
+setting `--cpu-period=50000` and `--cpu-quota=25000` (50% CPU).
+
+The default value for `--cpus` is `0.000`, which means there is no limit.
+
 For more information, see the [CFS documentation on bandwidth limiting](https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt).
 
 ### Cpuset constraint

integration-cli/docker_cli_run_unix_test.go

@@ -1409,3 +1409,23 @@ func (s *DockerDaemonSuite) TestRunWithDaemonDefaultSeccompProfile(c *check.C) {
 	c.Assert(err, check.NotNil)
 	c.Assert(out, checker.Contains, "Operation not permitted")
 }
+
+func (s *DockerSuite) TestRunWithNanoCPUs(c *check.C) {
+	testRequires(c, cpuCfsQuota, cpuCfsPeriod)
+
+	file1 := "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"
+	file2 := "/sys/fs/cgroup/cpu/cpu.cfs_period_us"
+	out, _ := dockerCmd(c, "run", "--cpus", "0.5", "--name", "test", "busybox", "sh", "-c", fmt.Sprintf("cat %s && cat %s", file1, file2))
+	c.Assert(strings.TrimSpace(out), checker.Equals, "50000\n100000")
+
+	out = inspectField(c, "test", "HostConfig.NanoCpus")
+	c.Assert(out, checker.Equals, "5e+08", check.Commentf("setting the Nano CPUs failed"))
+	out = inspectField(c, "test", "HostConfig.CpuQuota")
+	c.Assert(out, checker.Equals, "0", check.Commentf("CPU CFS quota should be 0"))
+	out = inspectField(c, "test", "HostConfig.CpuPeriod")
+	c.Assert(out, checker.Equals, "0", check.Commentf("CPU CFS period should be 0"))
+
+	out, _, err := dockerCmdWithError("run", "--cpus", "0.5", "--cpu-quota", "50000", "--cpu-period", "100000", "busybox", "sh")
+	c.Assert(err, check.NotNil)
+	c.Assert(out, checker.Contains, "Conflicting options: Nano CPUs and CPU Period cannot both be set")
+}

man/docker-create.1.md

@@ -19,6 +19,7 @@ docker-create - Create a new container
 [**--cpu-quota**[=*0*]]
 [**--cpu-rt-period**[=*0*]]
 [**--cpu-rt-runtime**[=*0*]]
+[**--cpus**[=*0.0*]]
 [**--cpuset-cpus**[=*CPUSET-CPUS*]]
 [**--cpuset-mems**[=*CPUSET-MEMS*]]
 [**--device**[=*[]*]]
@@ -154,6 +155,9 @@ two memory nodes.
 
    The sum of all runtimes across containers cannot exceed the amount allotted to the parent cgroup.
 
+**--cpus**=0.0
+   Number of CPUs. The default is *0.0*.
+
 **--device**=[]
    Add a host device to the container (e.g. --device=/dev/sdc:/dev/xvdc:rwm)
 

man/docker-run.1.md

@@ -19,6 +19,7 @@ docker-run - Run a command in a new container
 [**--cpu-quota**[=*0*]]
 [**--cpu-rt-period**[=*0*]]
 [**--cpu-rt-runtime**[=*0*]]
+[**--cpus**[=*0.0*]]
 [**--cpuset-cpus**[=*CPUSET-CPUS*]]
 [**--cpuset-mems**[=*CPUSET-MEMS*]]
 [**-d**|**--detach**]
@@ -208,6 +209,9 @@ to the quota you specify.
 
    The sum of all runtimes across containers cannot exceed the amount allotted to the parent cgroup.
 
+**--cpus**=0.0
+   Number of CPUs. The default is *0.0* which means no limit.
+
 **-d**, **--detach**=*true*|*false*
    Detached mode: run the container in the background and print the new container ID. The default is *false*.
 

opts/opts.go

@@ -2,6 +2,7 @@ package opts
 
 import (
 	"fmt"
+	"math/big"
 	"net"
 	"regexp"
 	"strings"
@@ -319,3 +320,35 @@ func (o *FilterOpt) Type() string {
 func (o *FilterOpt) Value() filters.Args {
 	return o.filter
 }
+
+// NanoCPUs is a type for fixed point fractional number.
+type NanoCPUs int64
+
+// String returns the string format of the number
+func (c *NanoCPUs) String() string {
+	return big.NewRat(c.Value(), 1e9).FloatString(3)
+}
+
+// Set sets the value of the NanoCPU by passing a string
+func (c *NanoCPUs) Set(value string) error {
+	cpu, ok := new(big.Rat).SetString(value)
+	if !ok {
+		return fmt.Errorf("Failed to parse %v as a rational number", value)
+	}
+	nano := cpu.Mul(cpu, big.NewRat(1e9, 1))
+	if !nano.IsInt() {
+		return fmt.Errorf("value is too precise")
+	}
+	*c = NanoCPUs(nano.Num().Int64())
+	return nil
+}
+
+// Type returns the type
+func (c *NanoCPUs) Type() string {
+	return "NanoCPUs"
+}
+
+// Value returns the value in int64
+func (c *NanoCPUs) Value() int64 {
+	return int64(*c)
+}

runconfig/opts/parse.go

@@ -79,6 +79,7 @@ type ContainerOptions struct {
 	cpuRealtimePeriod  int64
 	cpuRealtimeRuntime int64
 	cpuQuota           int64
+	cpus               opts.NanoCPUs
 	cpusetCpus         string
 	cpusetMems         string
 	blkioWeight        uint16
@@ -232,6 +233,7 @@ func AddFlags(flags *pflag.FlagSet) *ContainerOptions {
 	flags.Int64Var(&copts.cpuRealtimePeriod, "cpu-rt-period", 0, "Limit CPU real-time period in microseconds")
 	flags.Int64Var(&copts.cpuRealtimeRuntime, "cpu-rt-runtime", 0, "Limit CPU real-time runtime in microseconds")
 	flags.Int64VarP(&copts.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)")
+	flags.Var(&copts.cpus, "cpus", "Number of CPUs")
 	flags.Var(&copts.deviceReadBps, "device-read-bps", "Limit read rate (bytes per second) from a device")
 	flags.Var(&copts.deviceReadIOps, "device-read-iops", "Limit read rate (IO per second) from a device")
 	flags.Var(&copts.deviceWriteBps, "device-write-bps", "Limit write rate (bytes per second) to a device")
@@ -526,6 +528,7 @@ func Parse(flags *pflag.FlagSet, copts *ContainerOptions) (*container.Config, *c
 		MemorySwappiness:     &copts.swappiness,
 		KernelMemory:         kernelMemory,
 		OomKillDisable:       &copts.oomKillDisable,
+		NanoCPUs:             copts.cpus.Value(),
 		CPUPercent:           copts.cpuPercent,
 		CPUShares:            copts.cpuShares,
 		CPUPeriod:            copts.cpuPeriod,