Merge pull request #26342 from cpuguy83/20079_restore_volume_migrate

restore migrating pre-1.7.0 volumes

daemon/daemon.go

@@ -160,6 +160,12 @@ func (daemon *Daemon) restore() error {
 			continue
 		}
 
+		// verify that all volumes valid and have been migrated from the pre-1.7 layout
+		if err := daemon.verifyVolumesInfo(c); err != nil {
+			// don't skip the container due to error
+			logrus.Errorf("Failed to verify volumes for container '%s': %v", c.ID, err)
+		}
+
 		// The LogConfig.Type is empty if the container was created before docker 1.12 with default log driver.
 		// We should rewrite it to use the daemon defaults.
 		// Fixes https://github.com/docker/docker/issues/22536

daemon/daemon_solaris.go

@@ -165,3 +165,10 @@ func rootFSToAPIType(rootfs *image.RootFS) types.RootFS {
 func setupDaemonProcess(config *Config) error {
 	return nil
 }
+
+// verifyVolumesInfo is a no-op on solaris.
+// This is called during daemon initialization to migrate volumes from pre-1.7.
+// Solaris was not supported on pre-1.7 daemons.
+func (daemon *Daemon) verifyVolumesInfo(container *container.Container) error {
+	return nil
+}

daemon/daemon_unix_test.go

@@ -5,9 +5,14 @@ package daemon
 import (
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/docker/docker/container"
+	"github.com/docker/docker/volume"
+	"github.com/docker/docker/volume/drivers"
+	"github.com/docker/docker/volume/local"
+	"github.com/docker/docker/volume/store"
 	containertypes "github.com/docker/engine-api/types/container"
 )
 
@@ -197,3 +202,82 @@ func TestNetworkOptions(t *testing.T) {
 		t.Fatalf("Expected networkOptions error, got nil")
 	}
 }
+
+func TestMigratePre17Volumes(t *testing.T) {
+	rootDir, err := ioutil.TempDir("", "test-daemon-volumes")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(rootDir)
+
+	volumeRoot := filepath.Join(rootDir, "volumes")
+	err = os.MkdirAll(volumeRoot, 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	containerRoot := filepath.Join(rootDir, "containers")
+	cid := "1234"
+	err = os.MkdirAll(filepath.Join(containerRoot, cid), 0755)
+
+	vid := "5678"
+	vfsPath := filepath.Join(rootDir, "vfs", "dir", vid)
+	err = os.MkdirAll(vfsPath, 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	config := []byte(`
+		{
+			"ID": "` + cid + `",
+			"Volumes": {
+				"/foo": "` + vfsPath + `",
+				"/bar": "/foo",
+				"/quux": "/quux"
+			},
+			"VolumesRW": {
+				"/foo": true,
+				"/bar": true,
+				"/quux": false
+			}
+		}
+	`)
+
+	volStore, err := store.New(volumeRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	drv, err := local.New(volumeRoot, 0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	volumedrivers.Register(drv, volume.DefaultDriverName)
+
+	daemon := &Daemon{root: rootDir, repository: containerRoot, volumes: volStore}
+	err = ioutil.WriteFile(filepath.Join(containerRoot, cid, "config.v2.json"), config, 600)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c, err := daemon.load(cid)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := daemon.verifyVolumesInfo(c); err != nil {
+		t.Fatal(err)
+	}
+
+	expected := map[string]volume.MountPoint{
+		"/foo":  {Destination: "/foo", RW: true, Name: vid},
+		"/bar":  {Source: "/foo", Destination: "/bar", RW: true},
+		"/quux": {Source: "/quux", Destination: "/quux", RW: false},
+	}
+	for id, mp := range c.MountPoints {
+		x, exists := expected[id]
+		if !exists {
+			t.Fatal("volume not migrated")
+		}
+		if mp.Source != x.Source || mp.Destination != x.Destination || mp.RW != x.RW || mp.Name != x.Name {
+			t.Fatalf("got unexpected mountpoint, expected: %+v, got: %+v", x, mp)
+		}
+	}
+}

daemon/daemon_windows.go

@@ -434,3 +434,10 @@ func rootFSToAPIType(rootfs *image.RootFS) types.RootFS {
 func setupDaemonProcess(config *Config) error {
 	return nil
 }
+
+// verifyVolumesInfo is a no-op on windows.
+// This is called during daemon initialization to migrate volumes from pre-1.7.
+// volumes were not supported on windows pre-1.7
+func (daemon *Daemon) verifyVolumesInfo(container *container.Container) error {
+	return nil
+}

daemon/volumes_unix.go

@@ -3,12 +3,18 @@
 package daemon
 
 import (
+	"encoding/json"
 	"os"
+	"path/filepath"
 	"sort"
 	"strconv"
+	"strings"
 
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/volume"
+	"github.com/docker/docker/volume/drivers"
+	"github.com/docker/docker/volume/local"
+	"github.com/pkg/errors"
 )
 
 // setupMounts iterates through each of the mount points for a container and
@@ -85,3 +91,74 @@ func setBindModeIfNull(bind *volume.MountPoint) *volume.MountPoint {
 	}
 	return bind
 }
+
+// migrateVolume links the contents of a volume created pre Docker 1.7
+// into the location expected by the local driver.
+// It creates a symlink from DOCKER_ROOT/vfs/dir/VOLUME_ID to DOCKER_ROOT/volumes/VOLUME_ID/_container_data.
+// It preserves the volume json configuration generated pre Docker 1.7 to be able to
+// downgrade from Docker 1.7 to Docker 1.6 without losing volume compatibility.
+func migrateVolume(id, vfs string) error {
+	l, err := volumedrivers.GetDriver(volume.DefaultDriverName)
+	if err != nil {
+		return err
+	}
+
+	newDataPath := l.(*local.Root).DataPath(id)
+	fi, err := os.Stat(newDataPath)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	if fi != nil && fi.IsDir() {
+		return nil
+	}
+
+	return os.Symlink(vfs, newDataPath)
+}
+
+// verifyVolumesInfo ports volumes configured for the containers pre docker 1.7.
+// It reads the container configuration and creates valid mount points for the old volumes.
+func (daemon *Daemon) verifyVolumesInfo(container *container.Container) error {
+	// Inspect old structures only when we're upgrading from old versions
+	// to versions >= 1.7 and the MountPoints has not been populated with volumes data.
+	type volumes struct {
+		Volumes   map[string]string
+		VolumesRW map[string]bool
+	}
+	cfgPath, err := container.ConfigPath()
+	if err != nil {
+		return err
+	}
+	f, err := os.Open(cfgPath)
+	if err != nil {
+		return errors.Wrap(err, "could not open container config")
+	}
+	var cv volumes
+	if err := json.NewDecoder(f).Decode(&cv); err != nil {
+		return errors.Wrap(err, "could not decode container config")
+	}
+
+	if len(container.MountPoints) == 0 && len(cv.Volumes) > 0 {
+		for destination, hostPath := range cv.Volumes {
+			vfsPath := filepath.Join(daemon.root, "vfs", "dir")
+			rw := cv.VolumesRW != nil && cv.VolumesRW[destination]
+
+			if strings.HasPrefix(hostPath, vfsPath) {
+				id := filepath.Base(hostPath)
+				v, err := daemon.volumes.CreateWithRef(id, volume.DefaultDriverName, container.ID, nil, nil)
+				if err != nil {
+					return err
+				}
+				if err := migrateVolume(id, hostPath); err != nil {
+					return err
+				}
+				container.AddMountPointWithVolume(destination, v, true)
+			} else { // Bind mount
+				m := volume.MountPoint{Source: hostPath, Destination: destination, RW: rw}
+				container.MountPoints[destination] = &m
+			}
+		}
+		return container.ToDisk()
+	}
+	return nil
+}

integration-cli/docker_cli_daemon_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/docker/docker/pkg/integration/checker"
 	icmd "github.com/docker/docker/pkg/integration/cmd"
 	"github.com/docker/docker/pkg/mount"
+	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/go-units"
 	"github.com/docker/libnetwork/iptables"
 	"github.com/docker/libtrust"
@@ -2791,3 +2792,83 @@ func (s *DockerDaemonSuite) TestDaemonRestartSaveContainerExitCode(c *check.C) {
 	c.Assert(err, checker.IsNil)
 	c.Assert(out, checker.Equals, runError)
 }
+
+func (s *DockerDaemonSuite) TestDaemonBackcompatPre17Volumes(c *check.C) {
+	testRequires(c, SameHostDaemon)
+	d := s.d
+	err := d.StartWithBusybox()
+	c.Assert(err, checker.IsNil)
+
+	// hack to be able to side-load a container config
+	out, err := d.Cmd("create", "busybox:latest")
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+	id := strings.TrimSpace(out)
+
+	out, err = d.Cmd("inspect", "--type=image", "--format={{.ID}}", "busybox:latest")
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+	c.Assert(d.Stop(), checker.IsNil)
+	<-d.wait
+
+	imageID := strings.TrimSpace(out)
+	volumeID := stringid.GenerateNonCryptoID()
+	vfsPath := filepath.Join(d.root, "vfs", "dir", volumeID)
+	c.Assert(os.MkdirAll(vfsPath, 0755), checker.IsNil)
+
+	config := []byte(`
+		{
+			"ID": "` + id + `",
+			"Name": "hello",
+			"Driver": "` + d.storageDriver + `",
+			"Image": "` + imageID + `",
+			"Config": {"Image": "busybox:latest"},
+			"NetworkSettings": {},
+			"Volumes": {
+				"/bar":"/foo",
+				"/foo": "` + vfsPath + `",
+				"/quux":"/quux"
+			},
+			"VolumesRW": {
+				"/bar": true,
+				"/foo": true,
+				"/quux": false
+			}
+		}
+	`)
+
+	configPath := filepath.Join(d.root, "containers", id, "config.v2.json")
+	err = ioutil.WriteFile(configPath, config, 600)
+	err = d.Start()
+	c.Assert(err, checker.IsNil)
+
+	out, err = d.Cmd("inspect", "--type=container", "--format={{ json .Mounts }}", id)
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+	type mount struct {
+		Name        string
+		Source      string
+		Destination string
+		Driver      string
+		RW          bool
+	}
+
+	ls := []mount{}
+	err = json.NewDecoder(strings.NewReader(out)).Decode(&ls)
+	c.Assert(err, checker.IsNil)
+
+	expected := []mount{
+		{Source: "/foo", Destination: "/bar", RW: true},
+		{Name: volumeID, Destination: "/foo", RW: true},
+		{Source: "/quux", Destination: "/quux", RW: false},
+	}
+	c.Assert(ls, checker.HasLen, len(expected))
+
+	for _, m := range ls {
+		var matched bool
+		for _, x := range expected {
+			if m.Source == x.Source && m.Destination == x.Destination && m.RW == x.RW || m.Name != x.Name {
+				matched = true
+				break
+			}
+		}
+		c.Assert(matched, checker.True, check.Commentf("did find match for %+v", m))
+	}
+}