Add option to auto-configure blkdev for devmapper

Instead of forcing users to manually configure a block device to use
with devmapper, this gives the user the option to let the devmapper
driver configure a device for them.

Adds several new options to the devmapper storage-opts:

- dm.directlvm_device="" - path to the block device to configure for
  direct-lvm
- dm.thinp_percent=95 - sets the percentage of space to use for
  storage from the passed in block device
- dm.thinp_metapercent=1 - sets the percentage of space to for metadata
  storage from the passed in block device
- dm.thinp_autoextend_threshold=80 - sets the threshold for when `lvm`
  should automatically extend the thin pool as a percentage of the total
  storage space
- dm.thinp_autoextend_percent=20 - sets the percentage to increase the
  thin pool by when an autoextend is triggered.

Defaults are taken from
[here](https://docs.docker.com/engine/userguide/storagedriver/device-mapper-driver/#/configure-direct-lvm-mode-for-production)

The only option that is required is `dm.directlvm_device` for docker to
set everything up.

Changes to these settings are not currently supported and will error
out.
Future work could support allowing changes to these values.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

daemon/graphdriver/devmapper/README.md

@@ -5,7 +5,9 @@
 The device mapper graphdriver uses the device mapper thin provisioning
 module (dm-thinp) to implement CoW snapshots. The preferred model is
 to have a thin pool reserved outside of Docker and passed to the
-daemon via the `--storage-opt dm.thinpooldev` option.
+daemon via the `--storage-opt dm.thinpooldev` option. Alternatively,
+the device mapper graphdriver can setup a block device to handle this
+for you via the `--storage-opt dm.directlvm_device` option.
 
 As a fallback if no thin pool is provided, loopback files will be
 created.  Loopback is very slow, but can be used without any

daemon/graphdriver/devmapper/device_setup.go

@@ -0,0 +1,247 @@
+package devmapper
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"strings"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/pkg/errors"
+)
+
+type directLVMConfig struct {
+	Device              string
+	ThinpPercent        uint64
+	ThinpMetaPercent    uint64
+	AutoExtendPercent   uint64
+	AutoExtendThreshold uint64
+}
+
+var (
+	errThinpPercentMissing = errors.New("must set both `dm.thinp_percent` and `dm.thinp_metapercent` if either is specified")
+	errThinpPercentTooBig  = errors.New("combined `dm.thinp_percent` and `dm.thinp_metapercent` must not be greater than 100")
+	errMissingSetupDevice  = errors.New("must provide device path in `dm.setup_device` in order to configure direct-lvm")
+)
+
+func validateLVMConfig(cfg directLVMConfig) error {
+	if reflect.DeepEqual(cfg, directLVMConfig{}) {
+		return nil
+	}
+	if cfg.Device == "" {
+		return errMissingSetupDevice
+	}
+	if (cfg.ThinpPercent > 0 && cfg.ThinpMetaPercent == 0) || cfg.ThinpMetaPercent > 0 && cfg.ThinpPercent == 0 {
+		return errThinpPercentMissing
+	}
+
+	if cfg.ThinpPercent+cfg.ThinpMetaPercent > 100 {
+		return errThinpPercentTooBig
+	}
+	return nil
+}
+
+func checkDevAvailable(dev string) error {
+	lvmScan, err := exec.LookPath("lvmdiskscan")
+	if err != nil {
+		logrus.Debug("could not find lvmdiskscan")
+		return nil
+	}
+
+	out, err := exec.Command(lvmScan).CombinedOutput()
+	if err != nil {
+		logrus.WithError(err).Error(string(out))
+		return nil
+	}
+
+	if !bytes.Contains(out, []byte(dev)) {
+		return errors.Errorf("%s is not available for use with devicemapper", dev)
+	}
+	return nil
+}
+
+func checkDevInVG(dev string) error {
+	pvDisplay, err := exec.LookPath("pvdisplay")
+	if err != nil {
+		logrus.Debug("could not find pvdisplay")
+		return nil
+	}
+
+	out, err := exec.Command(pvDisplay, dev).CombinedOutput()
+	if err != nil {
+		logrus.WithError(err).Error(string(out))
+		return nil
+	}
+
+	scanner := bufio.NewScanner(bytes.NewReader(bytes.TrimSpace(out)))
+	for scanner.Scan() {
+		fields := strings.SplitAfter(strings.TrimSpace(scanner.Text()), "VG Name")
+		if len(fields) > 1 {
+			// got "VG Name" line"
+			vg := strings.TrimSpace(fields[1])
+			if len(vg) > 0 {
+				return errors.Errorf("%s is already part of a volume group %q: must remove this device from any volume group or provide a different device", dev, vg)
+			}
+			logrus.Error(fields)
+			break
+		}
+	}
+	return nil
+}
+
+func checkDevHasFS(dev string) error {
+	blkid, err := exec.LookPath("blkid")
+	if err != nil {
+		logrus.Debug("could not find blkid")
+		return nil
+	}
+
+	out, err := exec.Command(blkid, dev).CombinedOutput()
+	if err != nil {
+		logrus.WithError(err).Error(string(out))
+		return nil
+	}
+
+	fields := bytes.Fields(out)
+	for _, f := range fields {
+		kv := bytes.Split(f, []byte{'='})
+		if bytes.Equal(kv[0], []byte("TYPE")) {
+			v := bytes.Trim(kv[1], "\"")
+			if len(v) > 0 {
+				return errors.Errorf("%s has a filesystem already, use dm.directlvm_device_force=true if you want to wipe the device", dev)
+			}
+			return nil
+		}
+	}
+	return nil
+}
+
+func verifyBlockDevice(dev string, force bool) error {
+	if err := checkDevAvailable(dev); err != nil {
+		return err
+	}
+	if err := checkDevInVG(dev); err != nil {
+		return err
+	}
+
+	if force {
+		return nil
+	}
+
+	if err := checkDevHasFS(dev); err != nil {
+		return err
+	}
+	return nil
+}
+
+func readLVMConfig(root string) (directLVMConfig, error) {
+	var cfg directLVMConfig
+
+	p := filepath.Join(root, "setup-config.json")
+	b, err := ioutil.ReadFile(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return cfg, nil
+		}
+		return cfg, errors.Wrap(err, "error reading existing setup config")
+	}
+
+	// check if this is just an empty file, no need to produce a json error later if so
+	if len(b) == 0 {
+		return cfg, nil
+	}
+
+	err = json.Unmarshal(b, &cfg)
+	return cfg, errors.Wrap(err, "error unmarshaling previous device setup config")
+}
+
+func writeLVMConfig(root string, cfg directLVMConfig) error {
+	p := filepath.Join(root, "setup-config.json")
+	b, err := json.Marshal(cfg)
+	if err != nil {
+		return errors.Wrap(err, "error marshalling direct lvm config")
+	}
+	err = ioutil.WriteFile(p, b, 0600)
+	return errors.Wrap(err, "error writing direct lvm config to file")
+}
+
+func setupDirectLVM(cfg directLVMConfig) error {
+	pvCreate, err := exec.LookPath("pvcreate")
+	if err != nil {
+		return errors.Wrap(err, "error lookuping up command `pvcreate` while setting up direct lvm")
+	}
+
+	vgCreate, err := exec.LookPath("vgcreate")
+	if err != nil {
+		return errors.Wrap(err, "error lookuping up command `vgcreate` while setting up direct lvm")
+	}
+
+	lvCreate, err := exec.LookPath("lvcreate")
+	if err != nil {
+		return errors.Wrap(err, "error lookuping up command `lvcreate` while setting up direct lvm")
+	}
+
+	lvConvert, err := exec.LookPath("lvconvert")
+	if err != nil {
+		return errors.Wrap(err, "error lookuping up command `lvconvert` while setting up direct lvm")
+	}
+
+	lvChange, err := exec.LookPath("lvchange")
+	if err != nil {
+		return errors.Wrap(err, "error lookuping up command `lvchange` while setting up direct lvm")
+	}
+
+	if cfg.AutoExtendPercent == 0 {
+		cfg.AutoExtendPercent = 20
+	}
+
+	if cfg.AutoExtendThreshold == 0 {
+		cfg.AutoExtendThreshold = 80
+	}
+
+	if cfg.ThinpPercent == 0 {
+		cfg.ThinpPercent = 95
+	}
+	if cfg.ThinpMetaPercent == 0 {
+		cfg.ThinpMetaPercent = 1
+	}
+
+	out, err := exec.Command(pvCreate, "-f", cfg.Device).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	out, err = exec.Command(vgCreate, "docker", cfg.Device).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	out, err = exec.Command(lvCreate, "--wipesignatures", "y", "-n", "thinpool", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+	out, err = exec.Command(lvCreate, "--wipesignatures", "y", "-n", "thinpoolmeta", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	out, err = exec.Command(lvConvert, "-y", "--zero", "n", "-c", "512K", "--thinpool", "docker/thinpool", "--poolmetadata", "docker/thinpoolmeta").CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	profile := fmt.Sprintf("activation{\nthin_pool_autoextend_threshold=%d\nthin_pool_autoextend_percent=%d\n}", cfg.AutoExtendThreshold, cfg.AutoExtendPercent)
+	err = ioutil.WriteFile("/etc/lvm/profile/docker-thinpool.profile", []byte(profile), 0600)
+	if err != nil {
+		return errors.Wrap(err, "error writing docker thinp autoextend profile")
+	}
+
+	out, err = exec.Command(lvChange, "--metadataprofile", "docker-thinpool", "docker/thinpool").CombinedOutput()
+	return errors.Wrap(err, string(out))
+}

daemon/graphdriver/devmapper/deviceset.go

@@ -5,7 +5,6 @@ package devmapper
 import (
 	"bufio"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -13,6 +12,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"reflect"
 	"strconv"
 	"strings"
 	"sync"
@@ -29,6 +29,7 @@ import (
 	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/parsers"
 	units "github.com/docker/go-units"
+	"github.com/pkg/errors"
 
 	"github.com/opencontainers/runc/libcontainer/label"
 )
@@ -50,6 +51,7 @@ var (
 	enableDeferredDeletion              = false
 	userBaseSize                        = false
 	defaultMinFreeSpacePercent   uint32 = 10
+	lvmSetupConfigForce          bool
 )
 
 const deviceSetMetaFile string = "deviceset-metadata"
@@ -123,6 +125,7 @@ type DeviceSet struct {
 	gidMaps               []idtools.IDMap
 	minFreeSpacePercent   uint32 //min free space percentage in thinpool
 	xfsNospaceRetries     string // max retries when xfs receives ENOSPC
+	lvmSetupConfig        directLVMConfig
 }
 
 // DiskUsage contains information about disk usage and is used when reporting Status of a device.
@@ -1730,8 +1733,36 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
 		return err
 	}
 
-	// Set the device prefix from the device id and inode of the docker root dir
+	prevSetupConfig, err := readLVMConfig(devices.root)
+	if err != nil {
+		return err
+	}
 
+	if !reflect.DeepEqual(devices.lvmSetupConfig, directLVMConfig{}) {
+		if devices.thinPoolDevice != "" {
+			return errors.New("cannot setup direct-lvm when `dm.thinpooldev` is also specified")
+		}
+
+		if !reflect.DeepEqual(prevSetupConfig, devices.lvmSetupConfig) {
+			if !reflect.DeepEqual(prevSetupConfig, directLVMConfig{}) {
+				return errors.New("changing direct-lvm config is not supported")
+			}
+			logrus.WithField("storage-driver", "devicemapper").WithField("direct-lvm-config", devices.lvmSetupConfig).Debugf("Setting up direct lvm mode")
+			if err := verifyBlockDevice(devices.lvmSetupConfig.Device, lvmSetupConfigForce); err != nil {
+				return err
+			}
+			if err := setupDirectLVM(devices.lvmSetupConfig); err != nil {
+				return err
+			}
+			if err := writeLVMConfig(devices.root, devices.lvmSetupConfig); err != nil {
+				return err
+			}
+		}
+		devices.thinPoolDevice = "docker-thinpool"
+		logrus.WithField("storage-driver", "devicemapper").Debugf("Setting dm.thinpooldev to %q", devices.thinPoolDevice)
+	}
+
+	// Set the device prefix from the device id and inode of the docker root dir
 	st, err := os.Stat(devices.root)
 	if err != nil {
 		return fmt.Errorf("devmapper: Error looking up dir %s: %s", devices.root, err)
@@ -2605,6 +2636,7 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
 	}
 
 	foundBlkDiscard := false
+	var lvmSetupConfig directLVMConfig
 	for _, option := range options {
 		key, val, err := parsers.ParseKeyValueOpt(option)
 		if err != nil {
@@ -2699,11 +2731,60 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
 				return nil, err
 			}
 			devices.xfsNospaceRetries = val
+		case "dm.directlvm_device":
+			lvmSetupConfig.Device = val
+		case "dm.directlvm_device_force":
+			lvmSetupConfigForce, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, err
+			}
+		case "dm.thinp_percent":
+			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse `dm.thinp_percent=%s`", val)
+			}
+			if per >= 100 {
+				return nil, errors.New("dm.thinp_percent must be greater than 0 and less than 100")
+			}
+			lvmSetupConfig.ThinpPercent = per
+		case "dm.thinp_metapercent":
+			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse `dm.thinp_metapercent=%s`", val)
+			}
+			if per >= 100 {
+				return nil, errors.New("dm.thinp_metapercent must be greater than 0 and less than 100")
+			}
+			lvmSetupConfig.ThinpMetaPercent = per
+		case "dm.thinp_autoextend_percent":
+			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_percent=%s`", val)
+			}
+			if per > 100 {
+				return nil, errors.New("dm.thinp_autoextend_percent must be greater than 0 and less than 100")
+			}
+			lvmSetupConfig.AutoExtendPercent = per
+		case "dm.thinp_autoextend_threshold":
+			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_threshold=%s`", val)
+			}
+			if per > 100 {
+				return nil, errors.New("dm.thinp_autoextend_threshold must be greater than 0 and less than 100")
+			}
+			lvmSetupConfig.AutoExtendThreshold = per
 		default:
 			return nil, fmt.Errorf("devmapper: Unknown option %s\n", key)
 		}
 	}
 
+	if err := validateLVMConfig(lvmSetupConfig); err != nil {
+		return nil, err
+	}
+
+	devices.lvmSetupConfig = lvmSetupConfig
+
 	// By default, don't do blk discard hack on raw devices, its rarely useful and is expensive
 	if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") {
 		devices.doBlkDiscard = false

docs/reference/commandline/dockerd.md

@@ -322,6 +322,60 @@ not use loopback in production. Ensure your Engine daemon has a
 $ sudo dockerd --storage-opt dm.thinpooldev=/dev/mapper/thin-pool
 ```
 
+##### `dm.directlvm_device`
+
+As an alternative to providing a thin pool as above, Docker can setup a block
+device for you.
+
+###### Example:
+
+```bash
+$ sudo dockerd --storage-opt dm.directlvm_device=/dev/xvdf
+```
+
+##### `dm.thinp_percent`
+
+Sets the percentage of passed in block device to use for storage.
+
+###### Example:
+
+```bash
+$ sudo dockerd --storage-opt dm.thinp_percent=95
+```
+
+##### `dm.thinp_metapercent`
+
+Sets the percentage of the passed in block device to use for metadata storage.
+
+###### Example:
+
+```bash
+$ sudo dockerd --storage-opt dm.thinp_metapercent=1
+```
+
+##### `dm.thinp_autoextend_threshold`
+
+Sets the value of the percentage of space used before `lvm` attempts to
+autoextend the available space [100 = disabled]
+
+###### Example:
+
+```bash
+$ sudo dockerd --storage-opt dm.thinp_autoextend_threshold=80
+```
+
+##### `dm.thinp_autoextend_percent`
+
+Sets the value percentage value to increase the thin pool by when when `lvm`
+attempts to autoextend the available space [100 = disabled]
+
+###### Example:
+
+```bash
+$ sudo dockerd --storage-opt dm.thinp_autoextend_percent=20
+```
+
+
 ##### `dm.basesize`
 
 Specifies the size to use when creating the base device, which limits the

man/dockerd.8.md

@@ -418,6 +418,54 @@ Example use:
    $ dockerd \
          --storage-opt dm.thinpooldev=/dev/mapper/thin-pool
 
+#### dm.directlvm_device
+
+As an alternative to manually creating a thin pool as above, Docker can
+automatically configure a block device for you.
+
+Example use:
+
+   $ dockerd \
+         --storage-opt dm.directlvm_device=/dev/xvdf
+
+##### dm.thinp_percent
+
+Sets the percentage of passed in block device to use for storage.
+
+###### Example:
+
+   $ sudo dockerd \
+        --storage-opt dm.thinp_percent=95
+
+##### `dm.thinp_metapercent`
+
+Sets the percentage of the passed in block device to use for metadata storage.
+
+###### Example:
+
+   $ sudo dockerd \
+         --storage-opt dm.thinp_metapercent=1
+
+##### dm.thinp_autoextend_threshold
+
+Sets the value of the percentage of space used before `lvm` attempts to
+autoextend the available space [100 = disabled]
+
+###### Example:
+
+   $ sudo dockerd \
+         --storage-opt dm.thinp_autoextend_threshold=80
+
+##### dm.thinp_autoextend_percent
+
+Sets the value percentage value to increase the thin pool by when when `lvm`
+attempts to autoextend the available space [100 = disabled]
+
+###### Example:
+
+   $ sudo dockerd \
+         --storage-opt dm.thinp_autoextend_percent=20
+
 #### dm.basesize
 
 Specifies the size to use when creating the base device, which limits