Don't create source directory while the daemon is being shutdown, fix #30348
If a container mount the socket the daemon is listening on into container while the daemon is being shutdown, the socket will not exist on the host, then daemon will assume it's a directory and create it on the host, this will cause the daemon can't start next time. fix issue https://github.com/moby/moby/issues/30348 To reproduce this issue, you can add following code ``` --- a/daemon/oci_linux.go +++ b/daemon/oci_linux.go @@ -8,6 +8,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/Sirupsen/logrus" "github.com/docker/docker/container" @@ -666,7 +667,8 @@ func (daemon *Daemon) createSpec(c *container.Container) (*libcontainerd.Spec, e if err := daemon.setupIpcDirs(c); err != nil { return nil, err } - + fmt.Printf("===please stop the daemon===\n") + time.Sleep(time.Second * 2) ms, err := daemon.setupMounts(c) if err != nil { return nil, err ``` step1 run a container which has `--restart always` and `-v /var/run/docker.sock:/sock` ``` $ docker run -ti --restart always -v /var/run/docker.sock:/sock busybox / # ``` step2 exit the the container ``` / # exit ``` and kill the daemon when you see ``` ===please stop the daemon=== ``` in the daemon log The daemon can't restart again and fail with `can't create unix socket /var/run/docker.sock: is a directory`. Signed-off-by: Lei Jitang <leijitang@huawei.com>
This commit is contained in:
parent
916e9ad754
commit
7318eba5b2
6 changed files with 43 additions and 4 deletions
|
@ -155,6 +155,8 @@ func (cli *DaemonCli) start(opts daemonOptions) (err error) {
|
|||
api := apiserver.New(serverConfig)
|
||||
cli.api = api
|
||||
|
||||
var hosts []string
|
||||
|
||||
for i := 0; i < len(cli.Config.Hosts); i++ {
|
||||
var err error
|
||||
if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
|
||||
|
@ -186,6 +188,7 @@ func (cli *DaemonCli) start(opts daemonOptions) (err error) {
|
|||
}
|
||||
}
|
||||
logrus.Debugf("Listener created for HTTP on %s (%s)", proto, addr)
|
||||
hosts = append(hosts, protoAddrParts[1])
|
||||
api.Accept(addr, ls...)
|
||||
}
|
||||
|
||||
|
@ -213,6 +216,8 @@ func (cli *DaemonCli) start(opts daemonOptions) (err error) {
|
|||
return fmt.Errorf("Error starting daemon: %v", err)
|
||||
}
|
||||
|
||||
d.StoreHosts(hosts)
|
||||
|
||||
// validate after NewDaemon has restored enabled plugins. Dont change order.
|
||||
if err := validateAuthzPlugins(cli.Config.AuthorizationPlugins, pluginStore); err != nil {
|
||||
return fmt.Errorf("Error validating authorization plugin: %v", err)
|
||||
|
|
|
@ -116,6 +116,17 @@ type Daemon struct {
|
|||
|
||||
diskUsageRunning int32
|
||||
pruneRunning int32
|
||||
hosts map[string]bool // hosts stores the addresses the daemon is listening on
|
||||
}
|
||||
|
||||
// StoreHosts stores the addresses the daemon is listening on
|
||||
func (daemon *Daemon) StoreHosts(hosts []string) {
|
||||
if daemon.hosts == nil {
|
||||
daemon.hosts = make(map[string]bool)
|
||||
}
|
||||
for _, h := range hosts {
|
||||
daemon.hosts[h] = true
|
||||
}
|
||||
}
|
||||
|
||||
// HasExperimental returns whether the experimental features of the daemon are enabled or not
|
||||
|
|
|
@ -46,7 +46,8 @@ func (daemon *Daemon) StateChanged(id string, e libcontainerd.StateInfo) error {
|
|||
c.StreamConfig.Wait()
|
||||
c.Reset(false)
|
||||
|
||||
restart, wait, err := c.RestartManager().ShouldRestart(e.ExitCode, c.HasBeenManuallyStopped, time.Since(c.StartedAt))
|
||||
// If daemon is being shutdown, don't let the container restart
|
||||
restart, wait, err := c.RestartManager().ShouldRestart(e.ExitCode, daemon.IsShuttingDown() || c.HasBeenManuallyStopped, time.Since(c.StartedAt))
|
||||
if err == nil && restart {
|
||||
c.RestartCount++
|
||||
c.SetRestarting(platformConstructExitStatus(e))
|
||||
|
|
|
@ -6,6 +6,7 @@ package daemon
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
|
@ -42,8 +43,19 @@ func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, er
|
|||
if err := daemon.lazyInitializeVolume(c.ID, m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// If the daemon is being shutdown, we should not let a container start if it is trying to
|
||||
// mount the socket the daemon is listening on. During daemon shutdown, the socket
|
||||
// (/var/run/docker.sock by default) doesn't exist anymore causing the call to m.Setup to
|
||||
// create at directory instead. This in turn will prevent the daemon to restart.
|
||||
checkfunc := func(m *volume.MountPoint) error {
|
||||
if _, exist := daemon.hosts[m.Source]; exist && daemon.IsShuttingDown() {
|
||||
return fmt.Errorf("Could not mount %q to container while the daemon is shutting down", m.Source)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
rootUID, rootGID := daemon.GetRemappedUIDGID()
|
||||
path, err := m.Setup(c.MountLabel, rootUID, rootGID)
|
||||
path, err := m.Setup(c.MountLabel, rootUID, rootGID, checkfunc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -24,7 +24,7 @@ func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, er
|
|||
if err := daemon.lazyInitializeVolume(c.ID, mount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s, err := mount.Setup(c.MountLabel, 0, 0)
|
||||
s, err := mount.Setup(c.MountLabel, 0, 0, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -146,7 +146,9 @@ func (m *MountPoint) Cleanup() error {
|
|||
|
||||
// Setup sets up a mount point by either mounting the volume if it is
|
||||
// configured, or creating the source directory if supplied.
|
||||
func (m *MountPoint) Setup(mountLabel string, rootUID, rootGID int) (path string, err error) {
|
||||
// The, optional, checkFun parameter allows doing additional checking
|
||||
// before creating the source directory on the host.
|
||||
func (m *MountPoint) Setup(mountLabel string, rootUID, rootGID int, checkFun func(m *MountPoint) error) (path string, err error) {
|
||||
defer func() {
|
||||
if err == nil {
|
||||
if label.RelabelNeeded(m.Mode) {
|
||||
|
@ -181,6 +183,14 @@ func (m *MountPoint) Setup(mountLabel string, rootUID, rootGID int) (path string
|
|||
|
||||
// system.MkdirAll() produces an error if m.Source exists and is a file (not a directory),
|
||||
if m.Type == mounttypes.TypeBind {
|
||||
// Before creating the source directory on the host, invoke checkFun if it's not nil. One of
|
||||
// the use case is to forbid creating the daemon socket as a directory if the daemon is in
|
||||
// the process of shutting down.
|
||||
if checkFun != nil {
|
||||
if err := checkFun(m); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
// idtools.MkdirAllNewAs() produces an error if m.Source exists and is a file (not a directory)
|
||||
// also, makes sure that if the directory is created, the correct remapped rootUID/rootGID will own it
|
||||
if err := idtools.MkdirAllNewAs(m.Source, 0755, rootUID, rootGID); err != nil {
|
||||
|
|
Loading…
Reference in a new issue