Merge pull request #12139 from hqhq/hq_add_cpusest_mems2

add support for cpuset.mems

api/client/build.go

@@ -56,6 +56,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 	flMemorySwap := cmd.String([]string{"-memory-swap"}, "", "Total memory (memory + swap), '-1' to disable swap")
 	flCPUShares := cmd.Int64([]string{"c", "-cpu-shares"}, 0, "CPU shares (relative weight)")
 	flCPUSetCpus := cmd.String([]string{"-cpuset-cpus"}, "", "CPUs in which to allow execution (0-3, 0,1)")
+	flCPUSetMems := cmd.String([]string{"-cpuset-mems"}, "", "MEMs in which to allow execution (0-3, 0,1)")
 
 	cmd.Require(flag.Exact, 1)
 	cmd.ParseFlags(args, true)
@@ -278,6 +279,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 	}
 
 	v.Set("cpusetcpus", *flCPUSetCpus)
+	v.Set("cpusetmems", *flCPUSetMems)
 	v.Set("cpushares", strconv.FormatInt(*flCPUShares, 10))
 	v.Set("memory", strconv.FormatInt(memory, 10))
 	v.Set("memswap", strconv.FormatInt(memorySwap, 10))

api/server/server.go

@@ -1213,6 +1213,7 @@ func postBuild(eng *engine.Engine, version version.Version, w http.ResponseWrite
 	job.Setenv("memswap", r.FormValue("memswap"))
 	job.Setenv("memory", r.FormValue("memory"))
 	job.Setenv("cpusetcpus", r.FormValue("cpusetcpus"))
+	job.Setenv("cpusetmems", r.FormValue("cpusetmems"))
 	job.Setenv("cpushares", r.FormValue("cpushares"))
 
 	// Job cancellation. Note: not all job types support this.

builder/evaluator.go

@@ -124,6 +124,7 @@ type Builder struct {
 
 	// Set resource restrictions for build containers
 	cpuSetCpus string
+	cpuSetMems string
 	cpuShares  int64
 	memory     int64
 	memorySwap int64

builder/internals.go

@@ -541,6 +541,7 @@ func (b *Builder) create() (*daemon.Container, error) {
 	hostConfig := &runconfig.HostConfig{
 		CpuShares:  b.cpuShares,
 		CpusetCpus: b.cpuSetCpus,
+		CpusetMems: b.cpuSetMems,
 		Memory:     b.memory,
 		MemorySwap: b.memorySwap,
 	}

builder/job.go

@@ -63,6 +63,7 @@ func (b *BuilderJob) CmdBuild(job *engine.Job) error {
 		memorySwap     = job.GetenvInt64("memswap")
 		cpuShares      = job.GetenvInt64("cpushares")
 		cpuSetCpus     = job.Getenv("cpusetcpus")
+		cpuSetMems     = job.Getenv("cpusetmems")
 		authConfig     = &registry.AuthConfig{}
 		configFile     = &registry.ConfigFile{}
 		tag            string
@@ -153,6 +154,7 @@ func (b *BuilderJob) CmdBuild(job *engine.Job) error {
 		dockerfileName:  dockerfileName,
 		cpuShares:       cpuShares,
 		cpuSetCpus:      cpuSetCpus,
+		cpuSetMems:      cpuSetMems,
 		memory:          memory,
 		memorySwap:      memorySwap,
 		cancelled:       job.WaitCancelled(),

daemon/container.go

@@ -355,6 +355,7 @@ func populateCommand(c *Container, env []string) error {
 		MemorySwap: c.hostConfig.MemorySwap,
 		CpuShares:  c.hostConfig.CpuShares,
 		CpusetCpus: c.hostConfig.CpusetCpus,
+		CpusetMems: c.hostConfig.CpusetMems,
 		Rlimits:    rlimits,
 	}
 

daemon/execdriver/driver.go

@@ -110,6 +110,7 @@ type Resources struct {
 	MemorySwap int64            `json:"memory_swap"`
 	CpuShares  int64            `json:"cpu_shares"`
 	CpusetCpus string           `json:"cpuset_cpus"`
+	CpusetMems string           `json:"cpuset_mems"`
 	Rlimits    []*ulimit.Rlimit `json:"rlimits"`
 }
 
@@ -204,6 +205,7 @@ func SetupCgroups(container *configs.Config, c *Command) error {
 		container.Cgroups.MemoryReservation = c.Resources.Memory
 		container.Cgroups.MemorySwap = c.Resources.MemorySwap
 		container.Cgroups.CpusetCpus = c.Resources.CpusetCpus
+		container.Cgroups.CpusetMems = c.Resources.CpusetMems
 	}
 
 	return nil

daemon/execdriver/lxc/lxc_template.go

@@ -110,6 +110,9 @@ lxc.cgroup.cpu.shares = {{.Resources.CpuShares}}
 {{if .Resources.CpusetCpus}}
 lxc.cgroup.cpuset.cpus = {{.Resources.CpusetCpus}}
 {{end}}
+{{if .Resources.CpusetMems}}
+lxc.cgroup.cpuset.mems = {{.Resources.CpusetMems}}
+{{end}}
 {{end}}
 
 {{if .LxcConfig}}

docs/man/docker-create.1.md

@@ -13,6 +13,7 @@ docker-create - Create a new container
 [**--cap-drop**[=*[]*]]
 [**--cidfile**[=*CIDFILE*]]
 [**--cpuset-cpus**[=*CPUSET-CPUS*]]
+[**--cpuset-mems**[=*CPUSET-MEMS*]]
 [**--device**[=*[]*]]
 [**--dns-search**[=*[]*]]
 [**--dns**[=*[]*]]
@@ -74,6 +75,13 @@ IMAGE [COMMAND] [ARG...]
 **--cpuset-cpus**=""
    CPUs in which to allow execution (0-3, 0,1)
 
+**--cpuset-mems**=""
+   Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems.
+
+   If you have four memory nodes on your system (0-3), use `--cpuset-mems=0,1`
+then processes in your Docker container will only use memory from the first
+two memory nodes.
+
 **--device**=[]
    Add a host device to the container (e.g. --device=/dev/sdc:/dev/xvdc:rwm)
 

docs/man/docker-run.1.md

@@ -13,6 +13,7 @@ docker-run - Run a command in a new container
 [**--cap-drop**[=*[]*]]
 [**--cidfile**[=*CIDFILE*]]
 [**--cpuset-cpus**[=*CPUSET-CPUS*]]
+[**--cpuset-mems**[=*CPUSET-MEMS*]]
 [**-d**|**--detach**[=*false*]]
 [**--device**[=*[]*]]
 [**--dns-search**[=*[]*]]
@@ -134,6 +135,13 @@ division of CPU shares:
 **--cpuset-cpus**=""
    CPUs in which to allow execution (0-3, 0,1)
 
+**--cpuset-mems**=""
+   Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems.
+
+   If you have four memory nodes on your system (0-3), use `--cpuset-mems=0,1`
+then processes in your Docker container will only use memory from the first
+two memory nodes.
+
 **-d**, **--detach**=*true*|*false*
    Detached mode: run the container in the background and print the new container ID. The default is *false*.
 

docs/sources/reference/api/docker_remote_api_v1.19.md

@@ -147,6 +147,7 @@ Create a container
                "MemorySwap": 0,
                "CpuShares": 512,
                "CpusetCpus": "0,1",
+               "CpusetMems": "0,1",
                "PortBindings": { "22/tcp": [{ "HostPort": "11022" }] },
                "PublishAllPorts": false,
                "Privileged": false,
@@ -191,6 +192,7 @@ Json Parameters:
       (ie. the relative weight vs other containers).
 -   **Cpuset** - The same as CpusetCpus, but deprecated, please don't use.
 -   **CpusetCpus** - String value containing the cgroups CpusetCpus to use.
+-   **CpusetMems** - Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems.
 -   **AttachStdin** - Boolean value, attaches to stdin.
 -   **AttachStdout** - Boolean value, attaches to stdout.
 -   **AttachStderr** - Boolean value, attaches to stderr.
@@ -340,6 +342,7 @@ Return low-level information on the container `id`
 			"CapDrop": null,
 			"ContainerIDFile": "",
 			"CpusetCpus": "",
+			"CpusetMems": "",
 			"CpuShares": 0,
 			"Devices": [],
 			"Dns": null,

docs/sources/reference/commandline/cli.md

@@ -597,6 +597,7 @@ is returned by the `docker attach` command to its caller too:
       --memory-swap=""         Total memory (memory + swap), `-1` to disable swap
       -c, --cpu-shares         CPU Shares (relative weight)
       --cpuset-cpus=""         CPUs in which to allow execution, e.g. `0-3`, `0,1`
+      --cpuset-mems=""         MEMs in which to allow execution, e.g. `0-3`, `0,1`
 
 Builds Docker images from a Dockerfile and a "context". A build's context is
 the files located in the specified `PATH` or `URL`.  The build process can
@@ -892,6 +893,7 @@ Creates a new container.
       --cgroup-parent=""         Optional parent cgroup for the container
       --cidfile=""               Write the container ID to the file
       --cpuset-cpus=""           CPUs in which to allow execution (0-3, 0,1)
+      --cpuset-mems=""           Memory nodes (MEMs) in which to allow execution (0-3, 0,1)
       --device=[]                Add a host device to the container
       --dns=[]                   Set custom DNS servers
       --dns-search=[]            Set custom DNS search domains
@@ -1844,6 +1846,7 @@ To remove an image using its digest:
       --cap-drop=[]              Drop Linux capabilities
       --cidfile=""               Write the container ID to the file
       --cpuset-cpus=""           CPUs in which to allow execution (0-3, 0,1)
+      --cpuset-mems=""           Memory nodes (MEMs) in which to allow execution (0-3, 0,1)
       -d, --detach=false         Run container in background and print container ID
       --device=[]                Add a host device to the container
       --dns=[]                   Set custom DNS servers

docs/sources/reference/run.md

@@ -474,6 +474,7 @@ container:
     -memory-swap="": Total memory limit (memory + swap, format: <number><optional unit>, where unit = b, k, m or g)
     -c, --cpu-shares=0: CPU shares (relative weight)
     --cpuset-cpus="": CPUs in which to allow execution (0-3, 0,1)
+    --cpuset-mems="": Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems.
 
 ### Memory constraints
 
@@ -599,6 +600,21 @@ This means processes in container can be executed on cpu 1 and cpu 3.
 
 This means processes in container can be executed on cpu 0, cpu 1 and cpu 2.
 
+We can set mems in which to allow execution for containers. Only effective
+on NUMA systems.
+
+Examples:
+
+    $ docker run -ti --cpuset-mems="1,3" ubuntu:14.04 /bin/bash
+
+This example restricts the processes in the container to only use memory from
+memory nodes 1 and 3.
+
+    $ docker run -ti --cpuset-mems="0-2" ubuntu:14.04 /bin/bash
+
+This example restricts the processes in the container to only use memory from
+memory nodes 0, 1 and 2.
+
 ## Runtime privilege, Linux capabilities, and LXC configuration
 
     --cap-add: Add Linux capabilities

integration-cli/docker_cli_build_test.go

@@ -5555,7 +5555,7 @@ func TestBuildResourceConstraintsAreUsed(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cmd := exec.Command(dockerBinary, "build", "--no-cache", "--rm=false", "--memory=64m", "--memory-swap=-1", "--cpuset-cpus=0", "--cpu-shares=100", "-t", name, ".")
+	cmd := exec.Command(dockerBinary, "build", "--no-cache", "--rm=false", "--memory=64m", "--memory-swap=-1", "--cpuset-cpus=0", "--cpuset-mems=0", "--cpu-shares=100", "-t", name, ".")
 	cmd.Dir = ctx.Dir
 
 	out, _, err := runCommandWithOutput(cmd)
@@ -5573,6 +5573,7 @@ func TestBuildResourceConstraintsAreUsed(t *testing.T) {
 		Memory     float64 // Use float64 here since the json decoder sees it that way
 		MemorySwap int
 		CpusetCpus string
+		CpusetMems string
 		CpuShares  int
 	}
 
@@ -5586,9 +5587,9 @@ func TestBuildResourceConstraintsAreUsed(t *testing.T) {
 		t.Fatal(err, cfg)
 	}
 	mem := int64(c1.Memory)
-	if mem != 67108864 || c1.MemorySwap != -1 || c1.CpusetCpus != "0" || c1.CpuShares != 100 {
-		t.Fatalf("resource constraints not set properly:\nMemory: %d, MemSwap: %d, CpusetCpus: %s, CpuShares: %d",
-			mem, c1.MemorySwap, c1.CpusetCpus, c1.CpuShares)
+	if mem != 67108864 || c1.MemorySwap != -1 || c1.CpusetCpus != "0" || c1.CpusetMems != "0" || c1.CpuShares != 100 {
+		t.Fatalf("resource constraints not set properly:\nMemory: %d, MemSwap: %d, CpusetCpus: %s, CpusetMems: %s, CpuShares: %d",
+			mem, c1.MemorySwap, c1.CpusetCpus, c1.CpusetMems, c1.CpuShares)
 	}
 
 	// Make sure constraints aren't saved to image
@@ -5605,9 +5606,9 @@ func TestBuildResourceConstraintsAreUsed(t *testing.T) {
 		t.Fatal(err, cfg)
 	}
 	mem = int64(c2.Memory)
-	if mem == 67108864 || c2.MemorySwap == -1 || c2.CpusetCpus == "0" || c2.CpuShares == 100 {
-		t.Fatalf("resource constraints leaked from build:\nMemory: %d, MemSwap: %d, CpusetCpus: %s, CpuShares: %d",
-			mem, c2.MemorySwap, c2.CpusetCpus, c2.CpuShares)
+	if mem == 67108864 || c2.MemorySwap == -1 || c2.CpusetCpus == "0" || c2.CpusetMems == "0" || c2.CpuShares == 100 {
+		t.Fatalf("resource constraints leaked from build:\nMemory: %d, MemSwap: %d, CpusetCpus: %s, CpusetMems: %s, CpuShares: %d",
+			mem, c2.MemorySwap, c2.CpusetCpus, c2.CpusetMems, c2.CpuShares)
 	}
 
 	logDone("build - resource constraints applied")

integration-cli/docker_cli_run_test.go

@@ -1367,7 +1367,7 @@ func TestRunWithCpuset(t *testing.T) {
 
 	cmd := exec.Command(dockerBinary, "run", "--cpuset", "0", "busybox", "true")
 	if code, err := runCommand(cmd); err != nil || code != 0 {
-		t.Fatalf("container should run successfuly with cpuset of 0: %s", err)
+		t.Fatalf("container should run successfully with cpuset of 0: %s", err)
 	}
 
 	logDone("run - cpuset 0")
@@ -1378,12 +1378,23 @@ func TestRunWithCpusetCpus(t *testing.T) {
 
 	cmd := exec.Command(dockerBinary, "run", "--cpuset-cpus", "0", "busybox", "true")
 	if code, err := runCommand(cmd); err != nil || code != 0 {
-		t.Fatalf("container should run successfuly with cpuset-cpus of 0: %s", err)
+		t.Fatalf("container should run successfully with cpuset-cpus of 0: %s", err)
 	}
 
 	logDone("run - cpuset-cpus 0")
 }
 
+func TestRunWithCpusetMems(t *testing.T) {
+	defer deleteAllContainers()
+
+	cmd := exec.Command(dockerBinary, "run", "--cpuset-mems", "0", "busybox", "true")
+	if code, err := runCommand(cmd); err != nil || code != 0 {
+		t.Fatalf("container should run successfully with cpuset-mems of 0: %s", err)
+	}
+
+	logDone("run - cpuset-mems 0")
+}
+
 func TestRunDeviceNumbers(t *testing.T) {
 	defer deleteAllContainers()
 

runconfig/hostconfig.go

@@ -116,6 +116,7 @@ type HostConfig struct {
 	MemorySwap      int64  // Total memory usage (memory + swap); set `-1` to disable swap
 	CpuShares       int64  // CPU shares (relative weight vs. other containers)
 	CpusetCpus      string // CpusetCpus 0-2, 0,1
+	CpusetMems      string // CpusetMems 0-2, 0,1
 	Privileged      bool
 	PortBindings    nat.PortMap
 	Links           []string

runconfig/parse.go

@@ -64,6 +64,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe
 		flWorkingDir      = cmd.String([]string{"w", "-workdir"}, "", "Working directory inside the container")
 		flCpuShares       = cmd.Int64([]string{"c", "-cpu-shares"}, 0, "CPU shares (relative weight)")
 		flCpusetCpus      = cmd.String([]string{"#-cpuset", "-cpuset-cpus"}, "", "CPUs in which to allow execution (0-3, 0,1)")
+		flCpusetMems      = cmd.String([]string{"-cpuset-mems"}, "", "MEMs in which to allow execution (0-3, 0,1)")
 		flNetMode         = cmd.String([]string{"-net"}, "bridge", "Set the Network mode for the container")
 		flMacAddress      = cmd.String([]string{"-mac-address"}, "", "Container MAC address (e.g. 92:d0:c6:0a:29:33)")
 		flIpcMode         = cmd.String([]string{"-ipc"}, "", "IPC namespace to use")
@@ -313,6 +314,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe
 		MemorySwap:      MemorySwap,
 		CpuShares:       *flCpuShares,
 		CpusetCpus:      *flCpusetCpus,
+		CpusetMems:      *flCpusetMems,
 		Privileged:      *flPrivileged,
 		PortBindings:    portBindings,
 		Links:           flLinks.GetAll(),