beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add option to auto-configure blkdev for devmapper

Browse source 
Instead of forcing users to manually configure a block device to use
with devmapper, this gives the user the option to let the devmapper
driver configure a device for them.

Adds several new options to the devmapper storage-opts:

- dm.directlvm_device="" - path to the block device to configure for
  direct-lvm
- dm.thinp_percent=95 - sets the percentage of space to use for
  storage from the passed in block device
- dm.thinp_metapercent=1 - sets the percentage of space to for metadata
  storage from the passed in block device
- dm.thinp_autoextend_threshold=80 - sets the threshold for when `lvm`
  should automatically extend the thin pool as a percentage of the total
  storage space
- dm.thinp_autoextend_percent=20 - sets the percentage to increase the
  thin pool by when an autoextend is triggered.

Defaults are taken from
[here](https://docs.docker.com/engine/userguide/storagedriver/device-mapper-driver/#/configure-direct-lvm-mode-for-production)

The only option that is required is `dm.directlvm_device` for docker to
set everything up.

Changes to these settings are not currently supported and will error
out.
Future work could support allowing changes to these values.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>
This commit is contained in:
Brian Goff 2017-02-16 15:33:03 -05:00
parent dab6c7d4f2
commit 5ef07d79c4
5 changed files with 435 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
daemon/graphdriver/devmapper/README.md
View file

@ -5,7 +5,9 @@
The device mapper graphdriver uses the device mapper thin provisioning
module (dm-thinp) to implement CoW snapshots. The preferred model is
to have a thin pool reserved outside of Docker and passed to the
daemon via the `--storage-opt dm.thinpooldev` option.
daemon via the `--storage-opt dm.thinpooldev` option. Alternatively,
the device mapper graphdriver can setup a block device to handle this
for you via the `--storage-opt dm.directlvm_device` option.
As a fallback if no thin pool is provided, loopback files will be
created. Loopback is very slow, but can be used without any

247
daemon/graphdriver/devmapper/device_setup.go Normal file
View file

@ -0,0 +1,247 @@
package devmapper
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"github.com/Sirupsen/logrus"
"github.com/pkg/errors"
)
type directLVMConfig struct {
Device string
ThinpPercent uint64
ThinpMetaPercent uint64
AutoExtendPercent uint64
AutoExtendThreshold uint64
}
var (
errThinpPercentMissing = errors.New("must set both `dm.thinp_percent` and `dm.thinp_metapercent` if either is specified")
errThinpPercentTooBig = errors.New("combined `dm.thinp_percent` and `dm.thinp_metapercent` must not be greater than 100")
errMissingSetupDevice = errors.New("must provide device path in `dm.setup_device` in order to configure direct-lvm")
)
func validateLVMConfig(cfg directLVMConfig) error {
if reflect.DeepEqual(cfg, directLVMConfig{}) {
return nil
}
if cfg.Device == "" {
return errMissingSetupDevice
}
if (cfg.ThinpPercent > 0 && cfg.ThinpMetaPercent == 0) || cfg.ThinpMetaPercent > 0 && cfg.ThinpPercent == 0 {
return errThinpPercentMissing
}
if cfg.ThinpPercent+cfg.ThinpMetaPercent > 100 {
return errThinpPercentTooBig
}
return nil
}
func checkDevAvailable(dev string) error {
lvmScan, err := exec.LookPath("lvmdiskscan")
if err != nil {
logrus.Debug("could not find lvmdiskscan")
return nil
}
out, err := exec.Command(lvmScan).CombinedOutput()
if err != nil {
logrus.WithError(err).Error(string(out))
return nil
}
if !bytes.Contains(out, []byte(dev)) {
return errors.Errorf("%s is not available for use with devicemapper", dev)
}
return nil
}
func checkDevInVG(dev string) error {
pvDisplay, err := exec.LookPath("pvdisplay")
if err != nil {
logrus.Debug("could not find pvdisplay")
return nil
}
out, err := exec.Command(pvDisplay, dev).CombinedOutput()
if err != nil {
logrus.WithError(err).Error(string(out))
return nil
}
scanner := bufio.NewScanner(bytes.NewReader(bytes.TrimSpace(out)))
for scanner.Scan() {
fields := strings.SplitAfter(strings.TrimSpace(scanner.Text()), "VG Name")
if len(fields) > 1 {
// got "VG Name" line"
vg := strings.TrimSpace(fields[1])
if len(vg) > 0 {
return errors.Errorf("%s is already part of a volume group %q: must remove this device from any volume group or provide a different device", dev, vg)
}
logrus.Error(fields)
break
}
}
return nil
}
func checkDevHasFS(dev string) error {
blkid, err := exec.LookPath("blkid")
if err != nil {
logrus.Debug("could not find blkid")
return nil
}
out, err := exec.Command(blkid, dev).CombinedOutput()
if err != nil {
logrus.WithError(err).Error(string(out))
return nil
}
fields := bytes.Fields(out)
for _, f := range fields {
kv := bytes.Split(f, []byte{'='})
if bytes.Equal(kv[0], []byte("TYPE")) {
v := bytes.Trim(kv[1], "\"")
if len(v) > 0 {
return errors.Errorf("%s has a filesystem already, use dm.directlvm_device_force=true if you want to wipe the device", dev)
}
return nil
}
}
return nil
}
func verifyBlockDevice(dev string, force bool) error {
if err := checkDevAvailable(dev); err != nil {
return err
}
if err := checkDevInVG(dev); err != nil {
return err
}
if force {
return nil
}
if err := checkDevHasFS(dev); err != nil {
return err
}
return nil
}
func readLVMConfig(root string) (directLVMConfig, error) {
var cfg directLVMConfig
p := filepath.Join(root, "setup-config.json")
b, err := ioutil.ReadFile(p)
if err != nil {
if os.IsNotExist(err) {
return cfg, nil
}
return cfg, errors.Wrap(err, "error reading existing setup config")
}
// check if this is just an empty file, no need to produce a json error later if so
if len(b) == 0 {
return cfg, nil
}
err = json.Unmarshal(b, &cfg)
return cfg, errors.Wrap(err, "error unmarshaling previous device setup config")
}
func writeLVMConfig(root string, cfg directLVMConfig) error {
p := filepath.Join(root, "setup-config.json")
b, err := json.Marshal(cfg)
if err != nil {
return errors.Wrap(err, "error marshalling direct lvm config")
}
err = ioutil.WriteFile(p, b, 0600)
return errors.Wrap(err, "error writing direct lvm config to file")
}
func setupDirectLVM(cfg directLVMConfig) error {
pvCreate, err := exec.LookPath("pvcreate")
if err != nil {
return errors.Wrap(err, "error lookuping up command `pvcreate` while setting up direct lvm")
}
vgCreate, err := exec.LookPath("vgcreate")
if err != nil {
return errors.Wrap(err, "error lookuping up command `vgcreate` while setting up direct lvm")
}
lvCreate, err := exec.LookPath("lvcreate")
if err != nil {
return errors.Wrap(err, "error lookuping up command `lvcreate` while setting up direct lvm")
}
lvConvert, err := exec.LookPath("lvconvert")
if err != nil {
return errors.Wrap(err, "error lookuping up command `lvconvert` while setting up direct lvm")
}
lvChange, err := exec.LookPath("lvchange")
if err != nil {
return errors.Wrap(err, "error lookuping up command `lvchange` while setting up direct lvm")
}
if cfg.AutoExtendPercent == 0 {
cfg.AutoExtendPercent = 20
}
if cfg.AutoExtendThreshold == 0 {
cfg.AutoExtendThreshold = 80
}
if cfg.ThinpPercent == 0 {
cfg.ThinpPercent = 95
}
if cfg.ThinpMetaPercent == 0 {
cfg.ThinpMetaPercent = 1
}
out, err := exec.Command(pvCreate, "-f", cfg.Device).CombinedOutput()
if err != nil {
return errors.Wrap(err, string(out))
}
out, err = exec.Command(vgCreate, "docker", cfg.Device).CombinedOutput()
if err != nil {
return errors.Wrap(err, string(out))
}
out, err = exec.Command(lvCreate, "--wipesignatures", "y", "-n", "thinpool", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput()
if err != nil {
return errors.Wrap(err, string(out))
}
out, err = exec.Command(lvCreate, "--wipesignatures", "y", "-n", "thinpoolmeta", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput()
if err != nil {
return errors.Wrap(err, string(out))
}
out, err = exec.Command(lvConvert, "-y", "--zero", "n", "-c", "512K", "--thinpool", "docker/thinpool", "--poolmetadata", "docker/thinpoolmeta").CombinedOutput()
if err != nil {
return errors.Wrap(err, string(out))
}
profile := fmt.Sprintf("activation{\nthin_pool_autoextend_threshold=%d\nthin_pool_autoextend_percent=%d\n}", cfg.AutoExtendThreshold, cfg.AutoExtendPercent)
err = ioutil.WriteFile("/etc/lvm/profile/docker-thinpool.profile", []byte(profile), 0600)
if err != nil {
return errors.Wrap(err, "error writing docker thinp autoextend profile")
}
out, err = exec.Command(lvChange, "--metadataprofile", "docker-thinpool", "docker/thinpool").CombinedOutput()
return errors.Wrap(err, string(out))
}

85
daemon/graphdriver/devmapper/deviceset.go
View file

@ -5,7 +5,6 @@ package devmapper
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
@ -13,6 +12,7 @@ import (
"os/exec"
"path"
"path/filepath"
"reflect"
"strconv"
"strings"
"sync"
@ -29,6 +29,7 @@ import (
"github.com/docker/docker/pkg/mount"
"github.com/docker/docker/pkg/parsers"
units "github.com/docker/go-units"
"github.com/pkg/errors"
"github.com/opencontainers/runc/libcontainer/label"
)
@ -50,6 +51,7 @@ var (
enableDeferredDeletion = false
userBaseSize = false
defaultMinFreeSpacePercent uint32 = 10
lvmSetupConfigForce bool
)
const deviceSetMetaFile string = "deviceset-metadata"
@ -123,6 +125,7 @@ type DeviceSet struct {
gidMaps []idtools.IDMap
minFreeSpacePercent uint32 //min free space percentage in thinpool
xfsNospaceRetries string // max retries when xfs receives ENOSPC
lvmSetupConfig directLVMConfig
}
// DiskUsage contains information about disk usage and is used when reporting Status of a device.
@ -1730,8 +1733,36 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
return err
}
// Set the device prefix from the device id and inode of the docker root dir
prevSetupConfig, err := readLVMConfig(devices.root)
if err != nil {
return err
}
if !reflect.DeepEqual(devices.lvmSetupConfig, directLVMConfig{}) {
if devices.thinPoolDevice != "" {
return errors.New("cannot setup direct-lvm when `dm.thinpooldev` is also specified")
}
if !reflect.DeepEqual(prevSetupConfig, devices.lvmSetupConfig) {
if !reflect.DeepEqual(prevSetupConfig, directLVMConfig{}) {
return errors.New("changing direct-lvm config is not supported")
}
logrus.WithField("storage-driver", "devicemapper").WithField("direct-lvm-config", devices.lvmSetupConfig).Debugf("Setting up direct lvm mode")
if err := verifyBlockDevice(devices.lvmSetupConfig.Device, lvmSetupConfigForce); err != nil {
return err
}
if err := setupDirectLVM(devices.lvmSetupConfig); err != nil {
return err
}
if err := writeLVMConfig(devices.root, devices.lvmSetupConfig); err != nil {
return err
}
}
devices.thinPoolDevice = "docker-thinpool"
logrus.WithField("storage-driver", "devicemapper").Debugf("Setting dm.thinpooldev to %q", devices.thinPoolDevice)
}
// Set the device prefix from the device id and inode of the docker root dir
st, err := os.Stat(devices.root)
if err != nil {
return fmt.Errorf("devmapper: Error looking up dir %s: %s", devices.root, err)
@ -2605,6 +2636,7 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
}
foundBlkDiscard := false
var lvmSetupConfig directLVMConfig
for _, option := range options {
key, val, err := parsers.ParseKeyValueOpt(option)
if err != nil {
@ -2699,11 +2731,60 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
return nil, err
}
devices.xfsNospaceRetries = val
case "dm.directlvm_device":
lvmSetupConfig.Device = val
case "dm.directlvm_device_force":
lvmSetupConfigForce, err = strconv.ParseBool(val)
if err != nil {
return nil, err
}
case "dm.thinp_percent":
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
if err != nil {
return nil, errors.Wrapf(err, "could not parse `dm.thinp_percent=%s`", val)
}
if per >= 100 {
return nil, errors.New("dm.thinp_percent must be greater than 0 and less than 100")
}
lvmSetupConfig.ThinpPercent = per
case "dm.thinp_metapercent":
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
if err != nil {
return nil, errors.Wrapf(err, "could not parse `dm.thinp_metapercent=%s`", val)
}
if per >= 100 {
return nil, errors.New("dm.thinp_metapercent must be greater than 0 and less than 100")
}
lvmSetupConfig.ThinpMetaPercent = per
case "dm.thinp_autoextend_percent":
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
if err != nil {
return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_percent=%s`", val)
}
if per > 100 {
return nil, errors.New("dm.thinp_autoextend_percent must be greater than 0 and less than 100")
}
lvmSetupConfig.AutoExtendPercent = per
case "dm.thinp_autoextend_threshold":
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
if err != nil {
return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_threshold=%s`", val)
}
if per > 100 {
return nil, errors.New("dm.thinp_autoextend_threshold must be greater than 0 and less than 100")
}
lvmSetupConfig.AutoExtendThreshold = per
default:
return nil, fmt.Errorf("devmapper: Unknown option %s\n", key)
}
}
if err := validateLVMConfig(lvmSetupConfig); err != nil {
return nil, err
}
devices.lvmSetupConfig = lvmSetupConfig
// By default, don't do blk discard hack on raw devices, its rarely useful and is expensive
if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") {
devices.doBlkDiscard = false

54
docs/reference/commandline/dockerd.md
View file

@ -322,6 +322,60 @@ not use loopback in production. Ensure your Engine daemon has a
$ sudo dockerd --storage-opt dm.thinpooldev=/dev/mapper/thin-pool
```
##### `dm.directlvm_device`
As an alternative to providing a thin pool as above, Docker can setup a block
device for you.
###### Example:
```bash
$ sudo dockerd --storage-opt dm.directlvm_device=/dev/xvdf
```
##### `dm.thinp_percent`
Sets the percentage of passed in block device to use for storage.
###### Example:
```bash
$ sudo dockerd --storage-opt dm.thinp_percent=95
```
##### `dm.thinp_metapercent`
Sets the percentage of the passed in block device to use for metadata storage.
###### Example:
```bash
$ sudo dockerd --storage-opt dm.thinp_metapercent=1
```
##### `dm.thinp_autoextend_threshold`
Sets the value of the percentage of space used before `lvm` attempts to
autoextend the available space [100 = disabled]
###### Example:
```bash
$ sudo dockerd --storage-opt dm.thinp_autoextend_threshold=80
```
##### `dm.thinp_autoextend_percent`
Sets the value percentage value to increase the thin pool by when when `lvm`
attempts to autoextend the available space [100 = disabled]
###### Example:
```bash
$ sudo dockerd --storage-opt dm.thinp_autoextend_percent=20
```
##### `dm.basesize`
Specifies the size to use when creating the base device, which limits the

48
man/dockerd.8.md
View file

@ -418,6 +418,54 @@ Example use:
$ dockerd \
--storage-opt dm.thinpooldev=/dev/mapper/thin-pool
#### dm.directlvm_device
As an alternative to manually creating a thin pool as above, Docker can
automatically configure a block device for you.
Example use:
$ dockerd \
--storage-opt dm.directlvm_device=/dev/xvdf
##### dm.thinp_percent
Sets the percentage of passed in block device to use for storage.
###### Example:
$ sudo dockerd \
--storage-opt dm.thinp_percent=95
##### `dm.thinp_metapercent`
Sets the percentage of the passed in block device to use for metadata storage.
###### Example:
$ sudo dockerd \
--storage-opt dm.thinp_metapercent=1
##### dm.thinp_autoextend_threshold
Sets the value of the percentage of space used before `lvm` attempts to
autoextend the available space [100 = disabled]
###### Example:
$ sudo dockerd \
--storage-opt dm.thinp_autoextend_threshold=80
##### dm.thinp_autoextend_percent
Sets the value percentage value to increase the thin pool by when when `lvm`
attempts to autoextend the available space [100 = disabled]
###### Example:
$ sudo dockerd \
--storage-opt dm.thinp_autoextend_percent=20
#### dm.basesize
Specifies the size to use when creating the base device, which limits