From 5ef07d79c4712d5b1ff4f0c896932ea8902a129c Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 16 Feb 2017 15:33:03 -0500 Subject: [PATCH] Add option to auto-configure blkdev for devmapper Instead of forcing users to manually configure a block device to use with devmapper, this gives the user the option to let the devmapper driver configure a device for them. Adds several new options to the devmapper storage-opts: - dm.directlvm_device="" - path to the block device to configure for direct-lvm - dm.thinp_percent=95 - sets the percentage of space to use for storage from the passed in block device - dm.thinp_metapercent=1 - sets the percentage of space to for metadata storage from the passed in block device - dm.thinp_autoextend_threshold=80 - sets the threshold for when `lvm` should automatically extend the thin pool as a percentage of the total storage space - dm.thinp_autoextend_percent=20 - sets the percentage to increase the thin pool by when an autoextend is triggered. Defaults are taken from [here](https://docs.docker.com/engine/userguide/storagedriver/device-mapper-driver/#/configure-direct-lvm-mode-for-production) The only option that is required is `dm.directlvm_device` for docker to set everything up. Changes to these settings are not currently supported and will error out. Future work could support allowing changes to these values. Signed-off-by: Brian Goff --- daemon/graphdriver/devmapper/README.md | 4 +- daemon/graphdriver/devmapper/device_setup.go | 247 +++++++++++++++++++ daemon/graphdriver/devmapper/deviceset.go | 85 ++++++- docs/reference/commandline/dockerd.md | 54 ++++ man/dockerd.8.md | 48 ++++ 5 files changed, 435 insertions(+), 3 deletions(-) create mode 100644 daemon/graphdriver/devmapper/device_setup.go diff --git a/daemon/graphdriver/devmapper/README.md b/daemon/graphdriver/devmapper/README.md index bed07869a0..6594fa65f0 100644 --- a/daemon/graphdriver/devmapper/README.md +++ b/daemon/graphdriver/devmapper/README.md @@ -5,7 +5,9 @@ The device mapper graphdriver uses the device mapper thin provisioning module (dm-thinp) to implement CoW snapshots. The preferred model is to have a thin pool reserved outside of Docker and passed to the -daemon via the `--storage-opt dm.thinpooldev` option. +daemon via the `--storage-opt dm.thinpooldev` option. Alternatively, +the device mapper graphdriver can setup a block device to handle this +for you via the `--storage-opt dm.directlvm_device` option. As a fallback if no thin pool is provided, loopback files will be created. Loopback is very slow, but can be used without any diff --git a/daemon/graphdriver/devmapper/device_setup.go b/daemon/graphdriver/devmapper/device_setup.go new file mode 100644 index 0000000000..31eeae79f1 --- /dev/null +++ b/daemon/graphdriver/devmapper/device_setup.go @@ -0,0 +1,247 @@ +package devmapper + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + + "github.com/Sirupsen/logrus" + "github.com/pkg/errors" +) + +type directLVMConfig struct { + Device string + ThinpPercent uint64 + ThinpMetaPercent uint64 + AutoExtendPercent uint64 + AutoExtendThreshold uint64 +} + +var ( + errThinpPercentMissing = errors.New("must set both `dm.thinp_percent` and `dm.thinp_metapercent` if either is specified") + errThinpPercentTooBig = errors.New("combined `dm.thinp_percent` and `dm.thinp_metapercent` must not be greater than 100") + errMissingSetupDevice = errors.New("must provide device path in `dm.setup_device` in order to configure direct-lvm") +) + +func validateLVMConfig(cfg directLVMConfig) error { + if reflect.DeepEqual(cfg, directLVMConfig{}) { + return nil + } + if cfg.Device == "" { + return errMissingSetupDevice + } + if (cfg.ThinpPercent > 0 && cfg.ThinpMetaPercent == 0) || cfg.ThinpMetaPercent > 0 && cfg.ThinpPercent == 0 { + return errThinpPercentMissing + } + + if cfg.ThinpPercent+cfg.ThinpMetaPercent > 100 { + return errThinpPercentTooBig + } + return nil +} + +func checkDevAvailable(dev string) error { + lvmScan, err := exec.LookPath("lvmdiskscan") + if err != nil { + logrus.Debug("could not find lvmdiskscan") + return nil + } + + out, err := exec.Command(lvmScan).CombinedOutput() + if err != nil { + logrus.WithError(err).Error(string(out)) + return nil + } + + if !bytes.Contains(out, []byte(dev)) { + return errors.Errorf("%s is not available for use with devicemapper", dev) + } + return nil +} + +func checkDevInVG(dev string) error { + pvDisplay, err := exec.LookPath("pvdisplay") + if err != nil { + logrus.Debug("could not find pvdisplay") + return nil + } + + out, err := exec.Command(pvDisplay, dev).CombinedOutput() + if err != nil { + logrus.WithError(err).Error(string(out)) + return nil + } + + scanner := bufio.NewScanner(bytes.NewReader(bytes.TrimSpace(out))) + for scanner.Scan() { + fields := strings.SplitAfter(strings.TrimSpace(scanner.Text()), "VG Name") + if len(fields) > 1 { + // got "VG Name" line" + vg := strings.TrimSpace(fields[1]) + if len(vg) > 0 { + return errors.Errorf("%s is already part of a volume group %q: must remove this device from any volume group or provide a different device", dev, vg) + } + logrus.Error(fields) + break + } + } + return nil +} + +func checkDevHasFS(dev string) error { + blkid, err := exec.LookPath("blkid") + if err != nil { + logrus.Debug("could not find blkid") + return nil + } + + out, err := exec.Command(blkid, dev).CombinedOutput() + if err != nil { + logrus.WithError(err).Error(string(out)) + return nil + } + + fields := bytes.Fields(out) + for _, f := range fields { + kv := bytes.Split(f, []byte{'='}) + if bytes.Equal(kv[0], []byte("TYPE")) { + v := bytes.Trim(kv[1], "\"") + if len(v) > 0 { + return errors.Errorf("%s has a filesystem already, use dm.directlvm_device_force=true if you want to wipe the device", dev) + } + return nil + } + } + return nil +} + +func verifyBlockDevice(dev string, force bool) error { + if err := checkDevAvailable(dev); err != nil { + return err + } + if err := checkDevInVG(dev); err != nil { + return err + } + + if force { + return nil + } + + if err := checkDevHasFS(dev); err != nil { + return err + } + return nil +} + +func readLVMConfig(root string) (directLVMConfig, error) { + var cfg directLVMConfig + + p := filepath.Join(root, "setup-config.json") + b, err := ioutil.ReadFile(p) + if err != nil { + if os.IsNotExist(err) { + return cfg, nil + } + return cfg, errors.Wrap(err, "error reading existing setup config") + } + + // check if this is just an empty file, no need to produce a json error later if so + if len(b) == 0 { + return cfg, nil + } + + err = json.Unmarshal(b, &cfg) + return cfg, errors.Wrap(err, "error unmarshaling previous device setup config") +} + +func writeLVMConfig(root string, cfg directLVMConfig) error { + p := filepath.Join(root, "setup-config.json") + b, err := json.Marshal(cfg) + if err != nil { + return errors.Wrap(err, "error marshalling direct lvm config") + } + err = ioutil.WriteFile(p, b, 0600) + return errors.Wrap(err, "error writing direct lvm config to file") +} + +func setupDirectLVM(cfg directLVMConfig) error { + pvCreate, err := exec.LookPath("pvcreate") + if err != nil { + return errors.Wrap(err, "error lookuping up command `pvcreate` while setting up direct lvm") + } + + vgCreate, err := exec.LookPath("vgcreate") + if err != nil { + return errors.Wrap(err, "error lookuping up command `vgcreate` while setting up direct lvm") + } + + lvCreate, err := exec.LookPath("lvcreate") + if err != nil { + return errors.Wrap(err, "error lookuping up command `lvcreate` while setting up direct lvm") + } + + lvConvert, err := exec.LookPath("lvconvert") + if err != nil { + return errors.Wrap(err, "error lookuping up command `lvconvert` while setting up direct lvm") + } + + lvChange, err := exec.LookPath("lvchange") + if err != nil { + return errors.Wrap(err, "error lookuping up command `lvchange` while setting up direct lvm") + } + + if cfg.AutoExtendPercent == 0 { + cfg.AutoExtendPercent = 20 + } + + if cfg.AutoExtendThreshold == 0 { + cfg.AutoExtendThreshold = 80 + } + + if cfg.ThinpPercent == 0 { + cfg.ThinpPercent = 95 + } + if cfg.ThinpMetaPercent == 0 { + cfg.ThinpMetaPercent = 1 + } + + out, err := exec.Command(pvCreate, "-f", cfg.Device).CombinedOutput() + if err != nil { + return errors.Wrap(err, string(out)) + } + + out, err = exec.Command(vgCreate, "docker", cfg.Device).CombinedOutput() + if err != nil { + return errors.Wrap(err, string(out)) + } + + out, err = exec.Command(lvCreate, "--wipesignatures", "y", "-n", "thinpool", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput() + if err != nil { + return errors.Wrap(err, string(out)) + } + out, err = exec.Command(lvCreate, "--wipesignatures", "y", "-n", "thinpoolmeta", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput() + if err != nil { + return errors.Wrap(err, string(out)) + } + + out, err = exec.Command(lvConvert, "-y", "--zero", "n", "-c", "512K", "--thinpool", "docker/thinpool", "--poolmetadata", "docker/thinpoolmeta").CombinedOutput() + if err != nil { + return errors.Wrap(err, string(out)) + } + + profile := fmt.Sprintf("activation{\nthin_pool_autoextend_threshold=%d\nthin_pool_autoextend_percent=%d\n}", cfg.AutoExtendThreshold, cfg.AutoExtendPercent) + err = ioutil.WriteFile("/etc/lvm/profile/docker-thinpool.profile", []byte(profile), 0600) + if err != nil { + return errors.Wrap(err, "error writing docker thinp autoextend profile") + } + + out, err = exec.Command(lvChange, "--metadataprofile", "docker-thinpool", "docker/thinpool").CombinedOutput() + return errors.Wrap(err, string(out)) +} diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index ba845d4d01..ae1d7c4f7b 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -5,7 +5,6 @@ package devmapper import ( "bufio" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -13,6 +12,7 @@ import ( "os/exec" "path" "path/filepath" + "reflect" "strconv" "strings" "sync" @@ -29,6 +29,7 @@ import ( "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/parsers" units "github.com/docker/go-units" + "github.com/pkg/errors" "github.com/opencontainers/runc/libcontainer/label" ) @@ -50,6 +51,7 @@ var ( enableDeferredDeletion = false userBaseSize = false defaultMinFreeSpacePercent uint32 = 10 + lvmSetupConfigForce bool ) const deviceSetMetaFile string = "deviceset-metadata" @@ -123,6 +125,7 @@ type DeviceSet struct { gidMaps []idtools.IDMap minFreeSpacePercent uint32 //min free space percentage in thinpool xfsNospaceRetries string // max retries when xfs receives ENOSPC + lvmSetupConfig directLVMConfig } // DiskUsage contains information about disk usage and is used when reporting Status of a device. @@ -1730,8 +1733,36 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { return err } - // Set the device prefix from the device id and inode of the docker root dir + prevSetupConfig, err := readLVMConfig(devices.root) + if err != nil { + return err + } + if !reflect.DeepEqual(devices.lvmSetupConfig, directLVMConfig{}) { + if devices.thinPoolDevice != "" { + return errors.New("cannot setup direct-lvm when `dm.thinpooldev` is also specified") + } + + if !reflect.DeepEqual(prevSetupConfig, devices.lvmSetupConfig) { + if !reflect.DeepEqual(prevSetupConfig, directLVMConfig{}) { + return errors.New("changing direct-lvm config is not supported") + } + logrus.WithField("storage-driver", "devicemapper").WithField("direct-lvm-config", devices.lvmSetupConfig).Debugf("Setting up direct lvm mode") + if err := verifyBlockDevice(devices.lvmSetupConfig.Device, lvmSetupConfigForce); err != nil { + return err + } + if err := setupDirectLVM(devices.lvmSetupConfig); err != nil { + return err + } + if err := writeLVMConfig(devices.root, devices.lvmSetupConfig); err != nil { + return err + } + } + devices.thinPoolDevice = "docker-thinpool" + logrus.WithField("storage-driver", "devicemapper").Debugf("Setting dm.thinpooldev to %q", devices.thinPoolDevice) + } + + // Set the device prefix from the device id and inode of the docker root dir st, err := os.Stat(devices.root) if err != nil { return fmt.Errorf("devmapper: Error looking up dir %s: %s", devices.root, err) @@ -2605,6 +2636,7 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [ } foundBlkDiscard := false + var lvmSetupConfig directLVMConfig for _, option := range options { key, val, err := parsers.ParseKeyValueOpt(option) if err != nil { @@ -2699,11 +2731,60 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [ return nil, err } devices.xfsNospaceRetries = val + case "dm.directlvm_device": + lvmSetupConfig.Device = val + case "dm.directlvm_device_force": + lvmSetupConfigForce, err = strconv.ParseBool(val) + if err != nil { + return nil, err + } + case "dm.thinp_percent": + per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "could not parse `dm.thinp_percent=%s`", val) + } + if per >= 100 { + return nil, errors.New("dm.thinp_percent must be greater than 0 and less than 100") + } + lvmSetupConfig.ThinpPercent = per + case "dm.thinp_metapercent": + per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "could not parse `dm.thinp_metapercent=%s`", val) + } + if per >= 100 { + return nil, errors.New("dm.thinp_metapercent must be greater than 0 and less than 100") + } + lvmSetupConfig.ThinpMetaPercent = per + case "dm.thinp_autoextend_percent": + per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_percent=%s`", val) + } + if per > 100 { + return nil, errors.New("dm.thinp_autoextend_percent must be greater than 0 and less than 100") + } + lvmSetupConfig.AutoExtendPercent = per + case "dm.thinp_autoextend_threshold": + per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_threshold=%s`", val) + } + if per > 100 { + return nil, errors.New("dm.thinp_autoextend_threshold must be greater than 0 and less than 100") + } + lvmSetupConfig.AutoExtendThreshold = per default: return nil, fmt.Errorf("devmapper: Unknown option %s\n", key) } } + if err := validateLVMConfig(lvmSetupConfig); err != nil { + return nil, err + } + + devices.lvmSetupConfig = lvmSetupConfig + // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") { devices.doBlkDiscard = false diff --git a/docs/reference/commandline/dockerd.md b/docs/reference/commandline/dockerd.md index 3b40540a33..a93f263a78 100644 --- a/docs/reference/commandline/dockerd.md +++ b/docs/reference/commandline/dockerd.md @@ -322,6 +322,60 @@ not use loopback in production. Ensure your Engine daemon has a $ sudo dockerd --storage-opt dm.thinpooldev=/dev/mapper/thin-pool ``` +##### `dm.directlvm_device` + +As an alternative to providing a thin pool as above, Docker can setup a block +device for you. + +###### Example: + +```bash +$ sudo dockerd --storage-opt dm.directlvm_device=/dev/xvdf +``` + +##### `dm.thinp_percent` + +Sets the percentage of passed in block device to use for storage. + +###### Example: + +```bash +$ sudo dockerd --storage-opt dm.thinp_percent=95 +``` + +##### `dm.thinp_metapercent` + +Sets the percentage of the passed in block device to use for metadata storage. + +###### Example: + +```bash +$ sudo dockerd --storage-opt dm.thinp_metapercent=1 +``` + +##### `dm.thinp_autoextend_threshold` + +Sets the value of the percentage of space used before `lvm` attempts to +autoextend the available space [100 = disabled] + +###### Example: + +```bash +$ sudo dockerd --storage-opt dm.thinp_autoextend_threshold=80 +``` + +##### `dm.thinp_autoextend_percent` + +Sets the value percentage value to increase the thin pool by when when `lvm` +attempts to autoextend the available space [100 = disabled] + +###### Example: + +```bash +$ sudo dockerd --storage-opt dm.thinp_autoextend_percent=20 +``` + + ##### `dm.basesize` Specifies the size to use when creating the base device, which limits the diff --git a/man/dockerd.8.md b/man/dockerd.8.md index 56408089d4..515991399d 100644 --- a/man/dockerd.8.md +++ b/man/dockerd.8.md @@ -418,6 +418,54 @@ Example use: $ dockerd \ --storage-opt dm.thinpooldev=/dev/mapper/thin-pool +#### dm.directlvm_device + +As an alternative to manually creating a thin pool as above, Docker can +automatically configure a block device for you. + +Example use: + + $ dockerd \ + --storage-opt dm.directlvm_device=/dev/xvdf + +##### dm.thinp_percent + +Sets the percentage of passed in block device to use for storage. + +###### Example: + + $ sudo dockerd \ + --storage-opt dm.thinp_percent=95 + +##### `dm.thinp_metapercent` + +Sets the percentage of the passed in block device to use for metadata storage. + +###### Example: + + $ sudo dockerd \ + --storage-opt dm.thinp_metapercent=1 + +##### dm.thinp_autoextend_threshold + +Sets the value of the percentage of space used before `lvm` attempts to +autoextend the available space [100 = disabled] + +###### Example: + + $ sudo dockerd \ + --storage-opt dm.thinp_autoextend_threshold=80 + +##### dm.thinp_autoextend_percent + +Sets the value percentage value to increase the thin pool by when when `lvm` +attempts to autoextend the available space [100 = disabled] + +###### Example: + + $ sudo dockerd \ + --storage-opt dm.thinp_autoextend_percent=20 + #### dm.basesize Specifies the size to use when creating the base device, which limits