Merge pull request #22563 from mlaventure/cgroup-devices

Allow adding rules to cgroup devices.allow on container create/run

api/types/container/host_config.go

@@ -251,6 +251,7 @@ type Resources struct {
 	CpusetCpus           string          // CpusetCpus 0-2, 0,1
 	CpusetMems           string          // CpusetMems 0-2, 0,1
 	Devices              []DeviceMapping // List of devices to map inside the container
+	DeviceCgroupRules    []string        // List of rule to be added to the device cgroup
 	DiskQuota            int64           // Disk limit (in bytes)
 	KernelMemory         int64           // Kernel memory limit (in bytes)
 	MemoryReservation    int64           // Memory soft limit (in bytes)

cli/command/container/opts.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -21,6 +22,10 @@ import (
 	"github.com/spf13/pflag"
 )
 
+var (
+	deviceCgroupRuleRegexp = regexp.MustCompile("^[acb] ([0-9]+|\\*):([0-9]+|\\*) [rwm]{1,3}$")
+)
+
 // containerOptions is a data object with all the options for creating a container
 type containerOptions struct {
 	attach             opts.ListOpts
@@ -36,6 +41,7 @@ type containerOptions struct {
 	deviceWriteIOps    opts.ThrottledeviceOpt
 	env                opts.ListOpts
 	labels             opts.ListOpts
+	deviceCgroupRules  opts.ListOpts
 	devices            opts.ListOpts
 	ulimits            *opts.UlimitOpt
 	sysctls            *opts.MapOpts
@@ -127,6 +133,7 @@ func addFlags(flags *pflag.FlagSet) *containerOptions {
 		dns:               opts.NewListOpts(opts.ValidateIPAddress),
 		dnsOptions:        opts.NewListOpts(nil),
 		dnsSearch:         opts.NewListOpts(opts.ValidateDNSSearch),
+		deviceCgroupRules: opts.NewListOpts(validateDeviceCgroupRule),
 		deviceReadBps:     opts.NewThrottledeviceOpt(opts.ValidateThrottleBpsDevice),
 		deviceReadIOps:    opts.NewThrottledeviceOpt(opts.ValidateThrottleIOpsDevice),
 		deviceWriteBps:    opts.NewThrottledeviceOpt(opts.ValidateThrottleBpsDevice),
@@ -154,6 +161,7 @@ func addFlags(flags *pflag.FlagSet) *containerOptions {
 
 	// General purpose flags
 	flags.VarP(&copts.attach, "attach", "a", "Attach to STDIN, STDOUT or STDERR")
+	flags.Var(&copts.deviceCgroupRules, "device-cgroup-rule", "Add a rule to the cgroup allowed devices list")
 	flags.Var(&copts.devices, "device", "Add a host device to the container")
 	flags.VarP(&copts.env, "env", "e", "Set environment variables")
 	flags.Var(&copts.envFile, "env-file", "Read in a file of environment variables")
@@ -553,6 +561,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions) (*container.Config, *c
 		IOMaximumIOps:        copts.ioMaxIOps,
 		IOMaximumBandwidth:   uint64(maxIOBandwidth),
 		Ulimits:              copts.ulimits.GetList(),
+		DeviceCgroupRules:    copts.deviceCgroupRules.GetAll(),
 		Devices:              deviceMappings,
 	}
 
@@ -767,6 +776,17 @@ func parseDevice(device string) (container.DeviceMapping, error) {
 	return deviceMapping, nil
 }
 
+// validateDeviceCgroupRule validates a device cgroup rule string format
+// It will make sure 'val' is in the form:
+//    'type major:minor mode'
+func validateDeviceCgroupRule(val string) (string, error) {
+	if deviceCgroupRuleRegexp.MatchString(val) {
+		return val, nil
+	}
+
+	return val, fmt.Errorf("invalid device cgroup format '%s'", val)
+}
+
 // validDeviceMode checks if the mode for device is valid or not.
 // Valid mode is a composition of r (read), w (write), and m (mknod).
 func validDeviceMode(mode string) bool {

contrib/completion/bash/docker

@@ -1358,6 +1358,7 @@ _docker_container_run() {
 		--cpuset-mems
 		--cpu-shares -c
 		--device
+		--device-cgroup-rule
 		--device-read-bps
 		--device-read-iops
 		--device-write-bps

contrib/completion/fish/docker.fish

@@ -121,6 +121,7 @@ complete -c docker -A -f -n '__fish_seen_subcommand_from create' -l cap-drop -d
 complete -c docker -A -f -n '__fish_seen_subcommand_from create' -l cidfile -d 'Write the container ID to the file'
 complete -c docker -A -f -n '__fish_seen_subcommand_from create' -l cpuset -d 'CPUs in which to allow execution (0-3, 0,1)'
 complete -c docker -A -f -n '__fish_seen_subcommand_from create' -l device -d 'Add a host device to the container (e.g. --device=/dev/sdc:/dev/xvdc:rwm)'
+complete -c docker -A -f -n '__fish_seen_subcommand_from create' -l device-cgroup-rule -d 'Add a rule to the cgroup allowed devices list (e.g. --device-cgroup-rule="c 13:37 rwm")'
 complete -c docker -A -f -n '__fish_seen_subcommand_from create' -l dns -d 'Set custom DNS servers'
 complete -c docker -A -f -n '__fish_seen_subcommand_from create' -l dns-opt -d "Set custom DNS options (Use --dns-opt='' if you don't wish to set options)"
 complete -c docker -A -f -n '__fish_seen_subcommand_from create' -l dns-search -d "Set custom DNS search domains (Use --dns-search=. if you don't wish to set the search domain)"
@@ -312,6 +313,7 @@ complete -c docker -A -f -n '__fish_seen_subcommand_from run' -l cidfile -d 'Wri
 complete -c docker -A -f -n '__fish_seen_subcommand_from run' -l cpuset -d 'CPUs in which to allow execution (0-3, 0,1)'
 complete -c docker -A -f -n '__fish_seen_subcommand_from run' -s d -l detach -d 'Detached mode: run the container in the background and print the new container ID'
 complete -c docker -A -f -n '__fish_seen_subcommand_from run' -l device -d 'Add a host device to the container (e.g. --device=/dev/sdc:/dev/xvdc:rwm)'
+complete -c docker -A -f -n '__fish_seen_subcommand_from create' -l device-cgroup-rule -d 'Add a rule to the cgroup allowed devices list (e.g. --device-cgroup-rule="c 13:37 rwm")'
 complete -c docker -A -f -n '__fish_seen_subcommand_from run' -l dns -d 'Set custom DNS servers'
 complete -c docker -A -f -n '__fish_seen_subcommand_from run' -l dns-opt -d "Set custom DNS options (Use --dns-opt='' if you don't wish to set options)"
 complete -c docker -A -f -n '__fish_seen_subcommand_from run' -l dns-search -d "Set custom DNS search domains (Use --dns-search=. if you don't wish to set the search domain)"

contrib/completion/zsh/_docker

@@ -546,6 +546,7 @@ __docker_container_subcommand() {
         "($help)--cidfile=[Write the container ID to the file]:CID file:_files"
         "($help)--cpus=[Number of CPUs (default 0.000)]:cpus: "
         "($help)*--device=[Add a host device to the container]:device:_files"
+        "($help)*--device-cgroup-rule=[Add a rule to the cgroup allowed devices list]:device:cgroup: "
         "($help)*--device-read-bps=[Limit the read rate (bytes per second) from a device]:device:IO rate: "
         "($help)*--device-read-iops=[Limit the read rate (IO per second) from a device]:device:IO rate: "
         "($help)*--device-write-bps=[Limit the write rate (bytes per second) to a device]:device:IO rate: "

daemon/oci_linux.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -27,6 +28,10 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
+var (
+	deviceCgroupRuleRegex = regexp.MustCompile("^([acb]) ([0-9]+|\\*):([0-9]+|\\*) ([rwm]{1,3})$")
+)
+
 func setResources(s *specs.Spec, r containertypes.Resources) error {
 	weightDevices, err := getBlkioWeightDevices(r)
 	if err != nil {
@@ -106,6 +111,41 @@ func setDevices(s *specs.Spec, c *container.Container) error {
 			devs = append(devs, d...)
 			devPermissions = append(devPermissions, dPermissions...)
 		}
+
+		for _, deviceCgroupRule := range c.HostConfig.DeviceCgroupRules {
+			ss := deviceCgroupRuleRegex.FindAllStringSubmatch(deviceCgroupRule, -1)
+			if len(ss[0]) != 5 {
+				return fmt.Errorf("invalid device cgroup rule format: '%s'", deviceCgroupRule)
+			}
+			matches := ss[0]
+
+			dPermissions := specs.DeviceCgroup{
+				Allow:  true,
+				Type:   &matches[1],
+				Access: &matches[4],
+			}
+			if matches[2] == "*" {
+				major := int64(-1)
+				dPermissions.Major = &major
+			} else {
+				major, err := strconv.ParseInt(matches[2], 10, 64)
+				if err != nil {
+					return fmt.Errorf("invalid major value in device cgroup rule format: '%s'", deviceCgroupRule)
+				}
+				dPermissions.Major = &major
+			}
+			if matches[3] == "*" {
+				minor := int64(-1)
+				dPermissions.Minor = &minor
+			} else {
+				minor, err := strconv.ParseInt(matches[3], 10, 64)
+				if err != nil {
+					return fmt.Errorf("invalid minor value in device cgroup rule format: '%s'", deviceCgroupRule)
+				}
+				dPermissions.Minor = &minor
+			}
+			devPermissions = append(devPermissions, dPermissions)
+		}
 	}
 
 	s.Linux.Devices = append(s.Linux.Devices, devs...)

docs/reference/commandline/create.md

@@ -44,6 +44,7 @@ Options:
       --cpuset-cpus string          CPUs in which to allow execution (0-3, 0,1)
       --cpuset-mems string          MEMs in which to allow execution (0-3, 0,1)
       --device value                Add a host device to the container (default [])
+      --device-cgroup-rule value    Add a rule to the cgroup allowed devices list
       --device-read-bps value       Limit read rate (bytes per second) from a device (default [])
       --device-read-iops value      Limit read rate (IO per second) from a device (default [])
       --device-write-bps value      Limit write rate (bytes per second) to a device (default [])

docs/reference/commandline/run.md

@@ -48,6 +48,7 @@ Options:
   -d, --detach                      Run container in background and print container ID
       --detach-keys string          Override the key sequence for detaching a container
       --device value                Add a host device to the container (default [])
+      --device-cgroup-rule value    Add a rule to the cgroup allowed devices list
       --device-read-bps value       Limit read rate (bytes per second) from a device (default [])
       --device-read-iops value      Limit read rate (IO per second) from a device (default [])
       --device-write-bps value      Limit write rate (bytes per second) to a device (default [])

integration-cli/docker_cli_run_test.go

@@ -4415,3 +4415,17 @@ func (s *DockerSuite) TestRunHostnameInHostMode(c *check.C) {
 	out, _ := dockerCmd(c, "run", "--net=host", "--hostname=foobar", "busybox", "sh", "-c", `echo $HOSTNAME && hostname`)
 	c.Assert(strings.TrimSpace(out), checker.Equals, expectedOutput)
 }
+
+func (s *DockerSuite) TestRunAddDeviceCgroupRule(c *check.C) {
+	testRequires(c, DaemonIsLinux)
+
+	deviceRule := "c 7:128 rwm"
+
+	out, _ := dockerCmd(c, "run", "--rm", "busybox", "cat", "/sys/fs/cgroup/devices/devices.list")
+	if strings.Contains(out, deviceRule) {
+		c.Fatalf("%s shouldn't been in the device.list", deviceRule)
+	}
+
+	out, _ = dockerCmd(c, "run", "--rm", fmt.Sprintf("--device-cgroup-rule=%s", deviceRule), "busybox", "grep", deviceRule, "/sys/fs/cgroup/devices/devices.list")
+	c.Assert(strings.TrimSpace(out), checker.Equals, deviceRule)
+}

man/docker-run.1.md

@@ -27,6 +27,7 @@ docker-run - Run a command in a new container
 [**-d**|**--detach**]
 [**--detach-keys**[=*[]*]]
 [**--device**[=*[]*]]
+[**--device-cgroup-rule**[=*[]*]]
 [**--device-read-bps**[=*[]*]]
 [**--device-read-iops**[=*[]*]]
 [**--device-write-bps**[=*[]*]]
@@ -246,6 +247,16 @@ See **config-json(5)** for documentation on using a configuration file.
 **--device**=[]
    Add a host device to the container (e.g. --device=/dev/sdc:/dev/xvdc:rwm)
 
+**--device-cgroup-rule**=[]
+   Add a rule to the cgroup allowed devices list.
+   
+   The rule is expected to be in the format specified in the Linux kernel documentation (Documentation/cgroup-v1/devices.txt):
+     - type: `a` (all), `c` (char) or `b` (block)
+     - major and minor: either a number or `*` for all
+     - permission: a composition of `r` (read), `w` (write) and `m` (mknod)
+
+   Example: `c 1:3 mr`: allow for character device with major `1` and minor `3` to be created (`m`) and read (`r`)
+
 **--device-read-bps**=[]
    Limit read rate from a device (e.g. --device-read-bps=/dev/sda:1mb)
 

man/generate.go

@@ -62,6 +62,18 @@ func loadLongDescription(cmd *cobra.Command, path string) error {
 			return err
 		}
 		cmd.Long = string(content)
+
+		fullpath = filepath.Join(path, cmd.Name()+"-example.md")
+		if _, err := os.Stat(fullpath); err != nil {
+			continue
+		}
+
+		content, err = ioutil.ReadFile(fullpath)
+		if err != nil {
+			return err
+		}
+		cmd.Example = string(content)
+
 	}
 	return nil
 }

man/src/container/create-example.md

@@ -0,0 +1,35 @@
+### Specify isolation technology for container (--isolation)
+
+This option is useful in situations where you are running Docker containers on
+Windows. The `--isolation=<value>` option sets a container's isolation
+technology. On Linux, the only supported is the `default` option which uses
+Linux namespaces. On Microsoft Windows, you can specify these values:
+
+* `default`: Use the value specified by the Docker daemon's `--exec-opt` . If the `daemon` does not specify an isolation technology, Microsoft Windows uses `process` as its default value.
+* `process`: Namespace isolation only.
+* `hyperv`: Hyper-V hypervisor partition-based isolation.
+
+Specifying the `--isolation` flag without a value is the same as setting `--isolation="default"`.
+
+### Dealing with dynamically created devices (--device-cgroup-rule)
+
+Devices available to a container are assigned at creation time. The
+assigned devices will both be added to the cgroup.allow file and
+created into the container once it is run. This poses a problem when
+a new device needs to be added to running container.
+
+One of the solution is to add a more permissive rule to a container
+allowing it access to a wider range of devices. For example, supposing
+our container needs access to a character device with major `42` and
+any number of minor number (added as new devices appear), the
+following rule would be added:
+
+```
+docker create --device-cgroup-rule='c 42:* rmw' -name my-container my-image
+```
+
+Then, a user could ask `udev` to execute a script that would `docker exec my-container mknod newDevX c 42 <minor>`
+the required device when it is added.
+
+NOTE: initially present devices still need to be explicitely added to
+the create/run command

man/src/container/create.md

@@ -6,7 +6,7 @@ any point.
 
 The initial status of the container created with **docker create** is 'created'.
 
-# OPTIONS 
+### OPTIONS 
 
 The `CONTAINER-DIR` must be an absolute path such as `/src/docs`. The `HOST-DIR`
 can be an absolute path or a `name` value. A `name` value must start with an
@@ -82,18 +82,3 @@ change propagation properties of source mount. Say `/` is source mount for
 
 To disable automatic copying of data from the container path to the volume, use
 the `nocopy` flag. The `nocopy` flag can be set on bind mounts and named volumes.
-
-# EXAMPLES
-
-## Specify isolation technology for container (--isolation)
-
-This option is useful in situations where you are running Docker containers on
-Windows. The `--isolation=<value>` option sets a container's isolation
-technology. On Linux, the only supported is the `default` option which uses
-Linux namespaces. On Microsoft Windows, you can specify these values:
-
-* `default`: Use the value specified by the Docker daemon's `--exec-opt` . If the `daemon` does not specify an isolation technology, Microsoft Windows uses `process` as its default value.
-* `process`: Namespace isolation only.
-* `hyperv`: Hyper-V hypervisor partition-based isolation.
-
-Specifying the `--isolation` flag without a value is the same as setting `--isolation="default"`.

vendor.conf

@@ -125,7 +125,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.0
 github.com/pkg/errors 839d9e913e063e28dfd0e6c7b7512793e0a48be9
 
 # cli
-github.com/spf13/cobra v1.5 https://github.com/dnephin/cobra.git
+github.com/spf13/cobra v1.5.1 https://github.com/dnephin/cobra.git
 github.com/spf13/pflag dabebe21bf790f782ea4c7bbd2efc430de182afd
 github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
 github.com/flynn-archive/go-shlex 3f9db97f856818214da2e1057f8ad84803971cff

vendor/github.com/spf13/cobra/doc/man_docs.go

@@ -197,7 +197,7 @@ func genMan(cmd *cobra.Command, header *GenManHeader) []byte {
 	manPrintOptions(buf, cmd)
 	if len(cmd.Example) > 0 {
 		fmt.Fprintf(buf, "# EXAMPLE\n")
-		fmt.Fprintf(buf, "```\n%s\n```\n", cmd.Example)
+		fmt.Fprintf(buf, "\n%s\n\n", cmd.Example)
 	}
 	if hasSeeAlso(cmd) {
 		fmt.Fprintf(buf, "# SEE ALSO\n")