Merge pull request #20526 from tiborvass/1.10.2-cherrypicks

1.10.2 cherrypicks

CHANGELOG.md

@@ -5,6 +5,35 @@ information on the list of deprecated flags and APIs please have a look at
 https://docs.docker.com/misc/deprecated/ where target removal dates can also
 be found.
 
+## 1.10.2 (2016-02-22)
+
+### Runtime
+
+- Prevent systemd from deleting containers' cgroups when its configuration is reloaded [#20518](https://github.com/docker/docker/pull/20518)
+- Fix SELinux issues by disregarding `--read-only` when mounting `/dev/mqueue` [#20333](https://github.com/docker/docker/pull/20333)
+- Fix chown permissions used during `docker cp` when userns is used [#20446](https://github.com/docker/docker/pull/20446)
+- Fix configuration loading issue with all booleans defaulting to `true` [#20471](https://github.com/docker/docker/pull/20471)
+- Fix occasional panic with `docker logs -f` [#20522](https://github.com/docker/docker/pull/20522)
+
+### Distribution
+
+- Keep layer reference if deletion failed to avoid a badly inconsistent state [#20513](https://github.com/docker/docker/pull/20513)
+- Handle gracefully a corner case when canceling migration [#20372](https://github.com/docker/docker/pull/20372)
+- Fix docker import on compressed data [#20367](https://github.com/docker/docker/pull/20367)
+- Fix tar-split files corruption during migration that later cause docker push and docker save to fail [#20458](https://github.com/docker/docker/pull/20458)
+
+### Networking
+
+- Fix daemon crash if embedded DNS is sent garbage [#20510](https://github.com/docker/docker/pull/20510)
+
+### Volumes
+
+- Fix issue with multiple volume references with same name [#20381](https://github.com/docker/docker/pull/20381)
+
+### Security
+
+- Fix potential cache corruption and delegation conflict issues [#20523](https://github.com/docker/docker/pull/20523)
+
 ## 1.10.1 (2016-02-11)
 
 ### Runtime

daemon/archive.go

@@ -248,13 +248,13 @@ func (daemon *Daemon) containerExtractToDir(container *container.Container, path
 		return ErrRootFSReadOnly
 	}
 
+	uid, gid := daemon.GetRemappedUIDGID()
 	options := &archive.TarOptions{
-		ChownOpts: &archive.TarChownOptions{
-			UID: 0, GID: 0, // TODO: use config.User? Remap to userns root?
-		},
 		NoOverwriteDirNonDir: noOverwriteDirNonDir,
+		ChownOpts: &archive.TarChownOptions{
+			UID: uid, GID: gid, // TODO: should all ownership be set to root (either real or remapped)?
+		},
 	}
-
 	if err := chrootarchive.Untar(content, resolvedPath, options); err != nil {
 		return err
 	}

daemon/config.go

@@ -151,14 +151,20 @@ func parseClusterAdvertiseSettings(clusterStore, clusterAdvertise string) (strin
 }
 
 // ReloadConfiguration reads the configuration in the host and reloads the daemon and server.
-func ReloadConfiguration(configFile string, flags *flag.FlagSet, reload func(*Config)) {
+func ReloadConfiguration(configFile string, flags *flag.FlagSet, reload func(*Config)) error {
 	logrus.Infof("Got signal to reload configuration, reloading from: %s", configFile)
 	newConfig, err := getConflictFreeConfiguration(configFile, flags)
 	if err != nil {
-		logrus.Error(err)
-	} else {
-		reload(newConfig)
+		return err
 	}
+	reload(newConfig)
+	return nil
+}
+
+// boolValue is an interface that boolean value flags implement
+// to tell the command line how to make -name equivalent to -name=true.
+type boolValue interface {
+	IsBoolFlag() bool
 }
 
 // MergeDaemonConfigurations reads a configuration file,
@@ -203,6 +209,36 @@ func getConflictFreeConfiguration(configFile string, flags *flag.FlagSet) (*Conf
 			return nil, err
 		}
 
+		// Override flag values to make sure the values set in the config file with nullable values, like `false`,
+		// are not overriden by default truthy values from the flags that were not explicitly set.
+		// See https://github.com/docker/docker/issues/20289 for an example.
+		//
+		// TODO: Rewrite configuration logic to avoid same issue with other nullable values, like numbers.
+		namedOptions := make(map[string]interface{})
+		for key, value := range configSet {
+			f := flags.Lookup("-" + key)
+			if f == nil { // ignore named flags that don't match
+				namedOptions[key] = value
+				continue
+			}
+
+			if _, ok := f.Value.(boolValue); ok {
+				f.Value.Set(fmt.Sprintf("%v", value))
+			}
+		}
+		if len(namedOptions) > 0 {
+			// set also default for mergeVal flags that are boolValue at the same time.
+			flags.VisitAll(func(f *flag.Flag) {
+				if opt, named := f.Value.(opts.NamedOption); named {
+					v, set := namedOptions[opt.Name()]
+					_, boolean := f.Value.(boolValue)
+					if set && boolean {
+						f.Value.Set(fmt.Sprintf("%v", v))
+					}
+				}
+			})
+		}
+
 		config.valuesSet = configSet
 	}
 
@@ -242,14 +278,16 @@ func findConfigurationConflicts(config map[string]interface{}, flags *flag.FlagS
 
 	// 2. Discard values that implement NamedOption.
 	// Their configuration name differs from their flag name, like `labels` and `label`.
-	unknownNamedConflicts := func(f *flag.Flag) {
-		if namedOption, ok := f.Value.(opts.NamedOption); ok {
-			if _, valid := unknownKeys[namedOption.Name()]; valid {
-				delete(unknownKeys, namedOption.Name())
+	if len(unknownKeys) > 0 {
+		unknownNamedConflicts := func(f *flag.Flag) {
+			if namedOption, ok := f.Value.(opts.NamedOption); ok {
+				if _, valid := unknownKeys[namedOption.Name()]; valid {
+					delete(unknownKeys, namedOption.Name())
+				}
 			}
 		}
+		flags.VisitAll(unknownNamedConflicts)
 	}
-	flags.VisitAll(unknownNamedConflicts)
 
 	if len(unknownKeys) > 0 {
 		var unknown []string

daemon/execdriver/native/create.go

@@ -103,7 +103,7 @@ func (d *Driver) createContainer(c *execdriver.Command, hooks execdriver.Hooks)
 	if container.Readonlyfs {
 		for i := range container.Mounts {
 			switch container.Mounts[i].Destination {
-			case "/proc", "/dev", "/dev/pts":
+			case "/proc", "/dev", "/dev/pts", "/dev/mqueue":
 				continue
 			}
 			container.Mounts[i].Flags |= syscall.MS_RDONLY

daemon/import.go

@@ -11,6 +11,7 @@ import (
 	"github.com/docker/docker/dockerversion"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/httputils"
 	"github.com/docker/docker/pkg/progress"
 	"github.com/docker/docker/pkg/streamformatter"
@@ -24,13 +25,13 @@ import (
 // the repo and tag arguments, respectively.
 func (daemon *Daemon) ImportImage(src string, newRef reference.Named, msg string, inConfig io.ReadCloser, outStream io.Writer, config *container.Config) error {
 	var (
-		sf      = streamformatter.NewJSONStreamFormatter()
-		archive io.ReadCloser
-		resp    *http.Response
+		sf   = streamformatter.NewJSONStreamFormatter()
+		rc   io.ReadCloser
+		resp *http.Response
 	)
 
 	if src == "-" {
-		archive = inConfig
+		rc = inConfig
 	} else {
 		inConfig.Close()
 		u, err := url.Parse(src)
@@ -48,15 +49,20 @@ func (daemon *Daemon) ImportImage(src string, newRef reference.Named, msg string
 			return err
 		}
 		progressOutput := sf.NewProgressOutput(outStream, true)
-		archive = progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Importing")
+		rc = progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Importing")
 	}
 
-	defer archive.Close()
+	defer rc.Close()
 	if len(msg) == 0 {
 		msg = "Imported from " + src
 	}
+
+	inflatedLayerData, err := archive.DecompressStream(rc)
+	if err != nil {
+		return err
+	}
 	// TODO: support windows baselayer?
-	l, err := daemon.layerStore.Register(archive, "")
+	l, err := daemon.layerStore.Register(inflatedLayerData, "")
 	if err != nil {
 		return err
 	}

docker/daemon_test.go

@@ -291,3 +291,80 @@ func TestLoadDaemonConfigWithMapOptions(t *testing.T) {
 		t.Fatalf("expected log tag `test`, got %s", tag)
 	}
 }
+
+func TestLoadDaemonConfigWithTrueDefaultValues(t *testing.T) {
+	c := &daemon.Config{}
+	common := &cli.CommonFlags{}
+	flags := mflag.NewFlagSet("test", mflag.ContinueOnError)
+	flags.BoolVar(&c.EnableUserlandProxy, []string{"-userland-proxy"}, true, "")
+
+	f, err := ioutil.TempFile("", "docker-config-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := flags.ParseFlags([]string{}, false); err != nil {
+		t.Fatal(err)
+	}
+
+	configFile := f.Name()
+	f.Write([]byte(`{
+		"userland-proxy": false
+}`))
+	f.Close()
+
+	loadedConfig, err := loadDaemonCliConfig(c, flags, common, configFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if loadedConfig == nil {
+		t.Fatal("expected configuration, got nil")
+	}
+
+	if loadedConfig.EnableUserlandProxy {
+		t.Fatal("expected userland proxy to be disabled, got enabled")
+	}
+
+	// make sure reloading doesn't generate configuration
+	// conflicts after normalizing boolean values.
+	err = daemon.ReloadConfiguration(configFile, flags, func(reloadedConfig *daemon.Config) {
+		if reloadedConfig.EnableUserlandProxy {
+			t.Fatal("expected userland proxy to be disabled, got enabled")
+		}
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestLoadDaemonConfigWithTrueDefaultValuesLeaveDefaults(t *testing.T) {
+	c := &daemon.Config{}
+	common := &cli.CommonFlags{}
+	flags := mflag.NewFlagSet("test", mflag.ContinueOnError)
+	flags.BoolVar(&c.EnableUserlandProxy, []string{"-userland-proxy"}, true, "")
+
+	f, err := ioutil.TempFile("", "docker-config-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := flags.ParseFlags([]string{}, false); err != nil {
+		t.Fatal(err)
+	}
+
+	configFile := f.Name()
+	f.Write([]byte(`{}`))
+	f.Close()
+
+	loadedConfig, err := loadDaemonCliConfig(c, flags, common, configFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if loadedConfig == nil {
+		t.Fatal("expected configuration, got nil")
+	}
+
+	if !loadedConfig.EnableUserlandProxy {
+		t.Fatal("expected userland proxy to be enabled, got disabled")
+	}
+}

docker/daemon_unix.go

@@ -8,6 +8,7 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/Sirupsen/logrus"
 	apiserver "github.com/docker/docker/api/server"
 	"github.com/docker/docker/daemon"
 	"github.com/docker/docker/pkg/mflag"
@@ -59,7 +60,9 @@ func setupConfigReloadTrap(configFile string, flags *mflag.FlagSet, reload func(
 	signal.Notify(c, syscall.SIGHUP)
 	go func() {
 		for range c {
-			daemon.ReloadConfiguration(configFile, flags, reload)
+			if err := daemon.ReloadConfiguration(configFile, flags, reload); err != nil {
+				logrus.Error(err)
+			}
 		}
 	}()
 }

docker/daemon_windows.go

@@ -50,7 +50,9 @@ func setupConfigReloadTrap(configFile string, flags *mflag.FlagSet, reload func(
 			logrus.Debugf("Config reload - waiting signal at %s", ev)
 			for {
 				syscall.WaitForSingleObject(h, syscall.INFINITE)
-				daemon.ReloadConfiguration(configFile, flags, reload)
+				if err := daemon.ReloadConfiguration(configFile, flags, reload); err != nil {
+					logrus.Error(err)
+				}
 			}
 		}
 	}()

integration-cli/docker_cli_cp_to_container_unix_test.go

@@ -0,0 +1,39 @@
+// +build !windows
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/integration/checker"
+	"github.com/docker/docker/pkg/system"
+	"github.com/go-check/check"
+)
+
+// Check ownership is root, both in non-userns and userns enabled modes
+func (s *DockerSuite) TestCpCheckDestOwnership(c *check.C) {
+	testRequires(c, DaemonIsLinux, SameHostDaemon)
+	tmpVolDir := getTestDir(c, "test-cp-tmpvol")
+	containerID := makeTestContainer(c,
+		testContainerOptions{volumes: []string{fmt.Sprintf("%s:/tmpvol", tmpVolDir)}})
+
+	tmpDir := getTestDir(c, "test-cp-to-check-ownership")
+	defer os.RemoveAll(tmpDir)
+
+	makeTestContentInDir(c, tmpDir)
+
+	srcPath := cpPath(tmpDir, "file1")
+	dstPath := containerCpPath(containerID, "/tmpvol", "file1")
+
+	err := runDockerCp(c, srcPath, dstPath)
+	c.Assert(err, checker.IsNil)
+
+	stat, err := system.Stat(filepath.Join(tmpVolDir, "file1"))
+	c.Assert(err, checker.IsNil)
+	uid, gid, err := getRootUIDGID()
+	c.Assert(err, checker.IsNil)
+	c.Assert(stat.UID(), checker.Equals, uint32(uid), check.Commentf("Copied file not owned by container root UID"))
+	c.Assert(stat.GID(), checker.Equals, uint32(gid), check.Commentf("Copied file not owned by container root GID"))
+}

integration-cli/docker_cli_import_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"compress/gzip"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -59,6 +60,31 @@ func (s *DockerSuite) TestImportFile(c *check.C) {
 	c.Assert(out, checker.Equals, "", check.Commentf("command output should've been nothing."))
 }
 
+func (s *DockerSuite) TestImportGzipped(c *check.C) {
+	testRequires(c, DaemonIsLinux)
+	dockerCmd(c, "run", "--name", "test-import", "busybox", "true")
+
+	temporaryFile, err := ioutil.TempFile("", "exportImportTest")
+	c.Assert(err, checker.IsNil, check.Commentf("failed to create temporary file"))
+	defer os.Remove(temporaryFile.Name())
+
+	runCmd := exec.Command(dockerBinary, "export", "test-import")
+	w := gzip.NewWriter(temporaryFile)
+	runCmd.Stdout = w
+
+	_, err = runCommand(runCmd)
+	c.Assert(err, checker.IsNil, check.Commentf("failed to export a container"))
+	err = w.Close()
+	c.Assert(err, checker.IsNil, check.Commentf("failed to close gzip writer"))
+	temporaryFile.Close()
+	out, _ := dockerCmd(c, "import", temporaryFile.Name())
+	c.Assert(out, checker.Count, "\n", 1, check.Commentf("display is expected 1 '\\n' but didn't"))
+	image := strings.TrimSpace(out)
+
+	out, _ = dockerCmd(c, "run", "--rm", image, "true")
+	c.Assert(out, checker.Equals, "", check.Commentf("command output should've been nothing."))
+}
+
 func (s *DockerSuite) TestImportFileWithMessage(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	dockerCmd(c, "run", "--name", "test-import", "busybox", "true")

integration-cli/docker_utils.go

@@ -1720,3 +1720,20 @@ func runSleepingContainerInImage(c *check.C, image string, extraArgs ...string)
 	args = append(args, defaultSleepCommand...)
 	return dockerCmd(c, args...)
 }
+
+func getRootUIDGID() (int, int, error) {
+	uidgid := strings.Split(filepath.Base(dockerBasePath), ".")
+	if len(uidgid) == 1 {
+		//user namespace remapping is not turned on; return 0
+		return 0, 0, nil
+	}
+	uid, err := strconv.Atoi(uidgid[0])
+	if err != nil {
+		return 0, 0, err
+	}
+	gid, err := strconv.Atoi(uidgid[1])
+	if err != nil {
+		return 0, 0, err
+	}
+	return uid, gid, nil
+}

layer/layer_store.go

@@ -498,18 +498,21 @@ func (ls *layerStore) ReleaseRWLayer(l RWLayer) ([]Metadata, error) {
 
 	if err := ls.driver.Remove(m.mountID); err != nil {
 		logrus.Errorf("Error removing mounted layer %s: %s", m.name, err)
+		m.retakeReference(l)
 		return nil, err
 	}
 
 	if m.initID != "" {
 		if err := ls.driver.Remove(m.initID); err != nil {
 			logrus.Errorf("Error removing init layer %s: %s", m.name, err)
+			m.retakeReference(l)
 			return nil, err
 		}
 	}
 
 	if err := ls.store.RemoveMount(m.name); err != nil {
 		logrus.Errorf("Error removing mount metadata: %s: %s", m.name, err)
+		m.retakeReference(l)
 		return nil, err
 	}
 

layer/migration.go

@@ -127,6 +127,7 @@ func (ls *layerStore) checksumForGraphIDNoTarsplit(id, parent, newTarDataPath st
 	}
 	defer f.Close()
 	mfz := gzip.NewWriter(f)
+	defer mfz.Close()
 	metaPacker := storage.NewJSONPacker(mfz)
 
 	packerCounter := &packSizeCounter{metaPacker, &size}

layer/mounted_layer.go

@@ -96,6 +96,13 @@ func (ml *mountedLayer) deleteReference(ref RWLayer) error {
 	return nil
 }
 
+func (ml *mountedLayer) retakeReference(r RWLayer) {
+	if ref, ok := r.(*referencedRWLayer); ok {
+		ref.activityCount = 0
+		ml.references[ref] = ref
+	}
+}
+
 type referencedRWLayer struct {
 	*mountedLayer
 

man/docker-exec.1.md

@@ -27,10 +27,10 @@ container is unpaused, and then run
 
 # OPTIONS
 **-d**, **--detach**=*true*|*false*
-    Override the key sequence for detaching a container. Format is a single character `[a-Z]` or `ctrl-<value>` where `<value>` is one of: `a-z`, `@`, `^`, `[`, `,` or `_`.
+   Detached mode: run command in the background. The default is *false*.
 
 **--detach-keys**=""
-  Define the key sequence which detaches the container.
+  Override the key sequence for detaching a container. Format is a single character `[a-Z]` or `ctrl-<value>` where `<value>` is one of: `a-z`, `@`, `^`, `[`, `,` or `_`.
 
 **--help**
   Print usage statement

migrate/v1/migratev1.go

@@ -160,7 +160,12 @@ func calculateLayerChecksum(graphDir, id string, ls checksumCalculator) error {
 		return err
 	}
 
-	if err := ioutil.WriteFile(filepath.Join(graphDir, id, migrationDiffIDFileName), []byte(diffID), 0600); err != nil {
+	tmpFile := filepath.Join(graphDir, id, migrationDiffIDFileName+".tmp")
+	if err := ioutil.WriteFile(tmpFile, []byte(diffID), 0600); err != nil {
+		return err
+	}
+
+	if err := os.Rename(tmpFile, filepath.Join(graphDir, id, migrationDiffIDFileName)); err != nil {
 		return err
 	}
 
@@ -423,7 +428,11 @@ func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metad
 		history = parentImg.History
 	}
 
-	diffID, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationDiffIDFileName))
+	diffIDData, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationDiffIDFileName))
+	if err != nil {
+		return err
+	}
+	diffID, err := digest.ParseDigest(string(diffIDData))
 	if err != nil {
 		return err
 	}

volume/store/store.go

@@ -291,16 +291,14 @@ func (s *VolumeStore) Dereference(v volume.Volume, ref string) {
 
 	s.globalLock.Lock()
 	defer s.globalLock.Unlock()
-	refs, exists := s.refs[v.Name()]
-	if !exists {
-		return
-	}
+	var refs []string
 
-	for i, r := range refs {
-		if r == ref {
-			s.refs[v.Name()] = append(s.refs[v.Name()][:i], s.refs[v.Name()][i+1:]...)
+	for _, r := range s.refs[v.Name()] {
+		if r != ref {
+			refs = append(refs, r)
 		}
 	}
+	s.refs[v.Name()] = refs
 }
 
 // Refs gets the current list of refs for the given volume

volume/store/store_test.go

@@ -157,3 +157,22 @@ func TestFilterByUsed(t *testing.T) {
 		t.Fatalf("expected used volume fake1, got %s", used[0].Name())
 	}
 }
+
+func TestDerefMultipleOfSameRef(t *testing.T) {
+	volumedrivers.Register(vt.NewFakeDriver("fake"), "fake")
+
+	s := New()
+	v, err := s.CreateWithRef("fake1", "fake", "volReference", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := s.GetWithRef("fake1", "fake", "volReference"); err != nil {
+		t.Fatal(err)
+	}
+
+	s.Dereference(v, "volReference")
+	if err := s.Remove(v); err != nil {
+		t.Fatal(err)
+	}
+}