From d222bf097cd4c8121900c292f0f6ff243773c3c5 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Wed, 31 Aug 2022 16:12:30 -0400 Subject: [PATCH] daemon: reload runtimes w/o breaking containers The existing runtimes reload logic went to great lengths to replace the directory containing runtime wrapper scripts as atomically as possible within the limitations of the Linux filesystem ABI. Trouble is, atomically swapping the wrapper scripts directory solves the wrong problem! The runtime configuration is "locked in" when a container is started, including the path to the runC binary. If a container is started with a runtime which requires a daemon-managed wrapper script and then the daemon is reloaded with a config which no longer requires the wrapper script (i.e. some args -> no args, or the runtime is dropped from the config), that container would become unmanageable. Any attempts to stop, exec or otherwise perform lifecycle management operations on the container are likely to fail due to the wrapper script no longer existing at its original path. Atomically swapping the wrapper scripts is also incompatible with the read-copy-update paradigm for reloading configuration. A handler in the daemon could retain a reference to the pre-reload configuration for an indeterminate amount of time after the daemon configuration has been reloaded and updated. It is possible for the daemon to attempt to start a container using a deleted wrapper script if a request to run a container races a reload. Solve the problem of deleting referenced wrapper scripts by ensuring that all wrapper scripts are *immutable* for the lifetime of the daemon process. Any given runtime wrapper script must always exist with the same contents, no matter how many times the daemon config is reloaded, or what changes are made to the config. This is accomplished by using everyone's favourite design pattern: content-addressable storage. Each wrapper script file name is suffixed with the SHA-256 digest of its contents to (probabilistically) guarantee immutability without needing any concurrency control. Stale runtime wrapper scripts are only cleaned up on the next daemon restart. Split the derived runtimes configuration from the user-supplied configuration to have a place to store derived state without mutating the user-supplied configuration or exposing daemon internals in API struct types. Hold the derived state and the user-supplied configuration in a single struct value so that they can be updated as an atomic unit. Signed-off-by: Cory Snider --- api/types/types.go | 11 -- cmd/dockerd/daemon.go | 2 +- daemon/config/config_linux.go | 9 -- daemon/config/config_windows.go | 6 - daemon/container.go | 2 +- daemon/container_operations.go | 2 +- daemon/container_unix_test.go | 11 +- daemon/create.go | 6 +- daemon/daemon.go | 125 ++++++++-------- daemon/daemon_linux.go | 8 +- daemon/daemon_linux_test.go | 23 ++- daemon/daemon_test.go | 3 +- daemon/daemon_unix.go | 8 +- daemon/daemon_unix_test.go | 6 +- daemon/daemon_windows.go | 6 +- daemon/delete.go | 2 +- daemon/exec.go | 2 +- daemon/exec_linux_test.go | 5 +- daemon/info.go | 12 +- daemon/info_unix.go | 66 +++++---- daemon/inspect.go | 4 +- daemon/inspect_linux.go | 2 +- daemon/inspect_test.go | 7 +- daemon/monitor.go | 4 +- daemon/network.go | 6 +- daemon/oci_linux.go | 14 +- daemon/oci_linux_test.go | 16 +-- daemon/oci_windows.go | 4 +- daemon/prune.go | 2 +- daemon/reload.go | 28 ++-- daemon/reload_test.go | 2 +- daemon/reload_unix.go | 14 +- daemon/reload_windows.go | 2 +- daemon/restart.go | 3 +- daemon/runtime_unix.go | 245 +++++++++++++++++--------------- daemon/runtime_unix_test.go | 212 ++++++++++++++++++++------- daemon/runtime_windows.go | 12 +- daemon/start.go | 11 +- daemon/start_unix.go | 7 +- daemon/start_windows.go | 3 +- 40 files changed, 518 insertions(+), 395 deletions(-) diff --git a/api/types/types.go b/api/types/types.go index 04be8e513b..60bc14c1aa 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -16,7 +16,6 @@ import ( "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/api/types/volume" "github.com/docker/go-connections/nat" - "github.com/opencontainers/runtime-spec/specs-go/features" ) const ( @@ -657,16 +656,6 @@ type Runtime struct { Type string `json:"runtimeType,omitempty"` Options map[string]interface{} `json:"options,omitempty"` - - // This is exposed here only for internal use - ShimConfig *ShimConfig `json:"-"` - Features *features.Features `json:"-"` -} - -// ShimConfig is used by runtime to configure containerd shims -type ShimConfig struct { - Binary string - Opts interface{} } // DiskUsageObject represents an object type used for disk usage query filtering. diff --git a/cmd/dockerd/daemon.go b/cmd/dockerd/daemon.go index 4c80f5d379..00d0a5d1c6 100644 --- a/cmd/dockerd/daemon.go +++ b/cmd/dockerd/daemon.go @@ -268,7 +268,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { // Restart all autostart containers which has a swarm endpoint // and is not yet running now that we have successfully // initialized the cluster. - d.RestartSwarmContainers(cli.Config) + d.RestartSwarmContainers() logrus.Info("Daemon has completed initialization") diff --git a/daemon/config/config_linux.go b/daemon/config/config_linux.go index 87dc96c10a..da0f6c168d 100644 --- a/daemon/config/config_linux.go +++ b/daemon/config/config_linux.go @@ -81,15 +81,6 @@ type Config struct { Rootless bool `json:"rootless,omitempty"` } -// GetRuntime returns the runtime path and arguments for a given -// runtime name -func (conf *Config) GetRuntime(name string) *types.Runtime { - if rt, ok := conf.Runtimes[name]; ok { - return &rt - } - return nil -} - // GetAllRuntimes returns a copy of the runtimes map func (conf *Config) GetAllRuntimes() map[string]types.Runtime { return conf.Runtimes diff --git a/daemon/config/config_windows.go b/daemon/config/config_windows.go index 7958abaac1..d13ff00144 100644 --- a/daemon/config/config_windows.go +++ b/daemon/config/config_windows.go @@ -30,12 +30,6 @@ type Config struct { // for the Windows daemon.) } -// GetRuntime returns the runtime path and arguments for a given -// runtime name -func (conf *Config) GetRuntime(name string) *types.Runtime { - return nil -} - // GetAllRuntimes returns a copy of the runtimes map func (conf *Config) GetAllRuntimes() map[string]types.Runtime { return map[string]types.Runtime{} diff --git a/daemon/container.go b/daemon/container.go index 63b70d52e8..d189670545 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -235,7 +235,7 @@ func (daemon *Daemon) setHostConfig(container *container.Container, hostConfig * // verifyContainerSettings performs validation of the hostconfig and config // structures. -func (daemon *Daemon) verifyContainerSettings(daemonCfg *config.Config, hostConfig *containertypes.HostConfig, config *containertypes.Config, update bool) (warnings []string, err error) { +func (daemon *Daemon) verifyContainerSettings(daemonCfg *configStore, hostConfig *containertypes.HostConfig, config *containertypes.Config, update bool) (warnings []string, err error) { // First perform verification of settings common across all platforms. if err = validateContainerConfig(config); err != nil { return warnings, err diff --git a/daemon/container_operations.go b/daemon/container_operations.go index f814a19e16..f69646e190 100644 --- a/daemon/container_operations.go +++ b/daemon/container_operations.go @@ -1075,7 +1075,7 @@ func (daemon *Daemon) ConnectToNetwork(container *container.Container, idOrName } } } else { - if err := daemon.connectToNetwork(daemon.config(), container, idOrName, endpointConfig, true); err != nil { + if err := daemon.connectToNetwork(&daemon.config().Config, container, idOrName, endpointConfig, true); err != nil { return err } } diff --git a/daemon/container_unix_test.go b/daemon/container_unix_test.go index f001255915..6aab11ee2a 100644 --- a/daemon/container_unix_test.go +++ b/daemon/container_unix_test.go @@ -31,11 +31,14 @@ func TestContainerWarningHostAndPublishPorts(t *testing.T) { NetworkMode: "host", PortBindings: tc.ports, } - cs := &config.Config{} - configureRuntimes(cs) d := &Daemon{} - d.configStore.Store(cs) - wrns, err := d.verifyContainerSettings(cs, hostConfig, &containertypes.Config{}, false) + cfg, err := config.New() + assert.NilError(t, err) + configureRuntimes(cfg) + runtimes, err := setupRuntimes(cfg) + assert.NilError(t, err) + daemonCfg := &configStore{Config: *cfg, Runtimes: runtimes} + wrns, err := d.verifyContainerSettings(daemonCfg, hostConfig, &containertypes.Config{}, false) assert.NilError(t, err) assert.DeepEqual(t, tc.warnings, wrns) } diff --git a/daemon/create.go b/daemon/create.go index c6df361807..1751cb6012 100644 --- a/daemon/create.go +++ b/daemon/create.go @@ -57,7 +57,7 @@ func (daemon *Daemon) ContainerCreateIgnoreImagesArgsEscaped(ctx context.Context }) } -func (daemon *Daemon) containerCreate(ctx context.Context, daemonCfg *config.Config, opts createOpts) (containertypes.CreateResponse, error) { +func (daemon *Daemon) containerCreate(ctx context.Context, daemonCfg *configStore, opts createOpts) (containertypes.CreateResponse, error) { start := time.Now() if opts.params.Config == nil { return containertypes.CreateResponse{}, errdefs.InvalidParameter(errors.New("Config cannot be empty in order to create a container")) @@ -95,12 +95,12 @@ func (daemon *Daemon) containerCreate(ctx context.Context, daemonCfg *config.Con if opts.params.HostConfig == nil { opts.params.HostConfig = &containertypes.HostConfig{} } - err = daemon.adaptContainerSettings(daemonCfg, opts.params.HostConfig, opts.params.AdjustCPUShares) + err = daemon.adaptContainerSettings(&daemonCfg.Config, opts.params.HostConfig, opts.params.AdjustCPUShares) if err != nil { return containertypes.CreateResponse{Warnings: warnings}, errdefs.InvalidParameter(err) } - ctr, err := daemon.create(ctx, daemonCfg, opts) + ctr, err := daemon.create(ctx, &daemonCfg.Config, opts) if err != nil { return containertypes.CreateResponse{Warnings: warnings}, err } diff --git a/daemon/daemon.go b/daemon/daemon.go index 8fd62882a0..b4cc02b729 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -77,6 +77,12 @@ import ( "resenje.org/singleflight" ) +type configStore struct { + config.Config + + Runtimes runtimes +} + // Daemon holds information about the Docker daemon. type Daemon struct { id string @@ -85,7 +91,7 @@ type Daemon struct { containersReplica *container.ViewDB execCommands *container.ExecStore imageService ImageService - configStore atomic.Pointer[config.Config] + configStore atomic.Pointer[configStore] configReload sync.Mutex statsCollector *stats.Collector defaultLogConfig containertypes.LogConfig @@ -159,10 +165,10 @@ func (daemon *Daemon) StoreHosts(hosts []string) { // lifetime of an operation, the configuration pointer should be passed down the // call stack, like one would a [context.Context] value. Only the entrypoints // for operations, the outermost functions, should call this function. -func (daemon *Daemon) config() *config.Config { +func (daemon *Daemon) config() *configStore { cfg := daemon.configStore.Load() if cfg == nil { - return &config.Config{} + return &configStore{} } return cfg } @@ -247,7 +253,7 @@ type layerAccessor interface { GetLayerByID(cid string) (layer.RWLayer, error) } -func (daemon *Daemon) restore(cfg *config.Config) error { +func (daemon *Daemon) restore(cfg *configStore) error { var mapLock sync.Mutex containers := make(map[string]*container.Container) @@ -467,7 +473,7 @@ func (daemon *Daemon) restore(cfg *config.Config) error { c.ResetRestartManager(false) if !c.HostConfig.NetworkMode.IsContainer() && c.IsRunning() { - options, err := daemon.buildSandboxOptions(cfg, c) + options, err := daemon.buildSandboxOptions(&cfg.Config, c) if err != nil { logger(c).WithError(err).Warn("failed to build sandbox option to restore container") } @@ -523,7 +529,7 @@ func (daemon *Daemon) restore(cfg *config.Config) error { // // Note that we cannot initialize the network controller earlier, as it // needs to know if there's active sandboxes (running containers). - if err = daemon.initNetworkController(cfg, activeSandboxes); err != nil { + if err = daemon.initNetworkController(&cfg.Config, activeSandboxes); err != nil { return fmt.Errorf("Error initializing network controller: %v", err) } @@ -586,7 +592,7 @@ func (daemon *Daemon) restore(cfg *config.Config) error { go func(cid string) { _ = sem.Acquire(context.Background(), 1) - if err := daemon.containerRm(cfg, cid, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil { + if err := daemon.containerRm(&cfg.Config, cid, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil { logrus.WithField("container", cid).WithError(err).Error("failed to remove container") } @@ -634,9 +640,11 @@ func (daemon *Daemon) restore(cfg *config.Config) error { // RestartSwarmContainers restarts any autostart container which has a // swarm endpoint. -func (daemon *Daemon) RestartSwarmContainers(cfg *config.Config) { - ctx := context.Background() +func (daemon *Daemon) RestartSwarmContainers() { + daemon.restartSwarmContainers(context.Background(), daemon.config()) +} +func (daemon *Daemon) restartSwarmContainers(ctx context.Context, cfg *configStore) { // parallelLimit is the maximum number of parallel startup jobs that we // allow (this is the limited used for all startup semaphores). The multipler // (128) was chosen after some fairly significant benchmarking -- don't change @@ -806,11 +814,23 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S os.Setenv("TMPDIR", realTmp) } + if err := initRuntimesDir(config); err != nil { + return nil, err + } + runtimes, err := setupRuntimes(config) + if err != nil { + return nil, err + } + d := &Daemon{ PluginStore: pluginStore, startupDone: make(chan struct{}), } - d.configStore.Store(config) + configStore := &configStore{ + Config: *config, + Runtimes: runtimes, + } + d.configStore.Store(configStore) // TEST_INTEGRATION_USE_SNAPSHOTTER is used for integration tests only. if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" { @@ -830,27 +850,27 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } }() - if err := d.setGenericResources(config); err != nil { + if err := d.setGenericResources(&configStore.Config); err != nil { return nil, err } // set up SIGUSR1 handler on Unix-like systems, or a Win32 global event // on Windows to dump Go routine stacks - stackDumpDir := config.Root - if execRoot := config.GetExecRoot(); execRoot != "" { + stackDumpDir := configStore.Root + if execRoot := configStore.GetExecRoot(); execRoot != "" { stackDumpDir = execRoot } d.setupDumpStackTrap(stackDumpDir) - if err := d.setupSeccompProfile(config); err != nil { + if err := d.setupSeccompProfile(&configStore.Config); err != nil { return nil, err } // Set the default isolation mode (only applicable on Windows) - if err := d.setDefaultIsolation(config); err != nil { + if err := d.setDefaultIsolation(&configStore.Config); err != nil { return nil, fmt.Errorf("error setting default isolation mode: %v", err) } - if err := configureMaxThreads(config); err != nil { + if err := configureMaxThreads(&configStore.Config); err != nil { logrus.Warnf("Failed to configure golang's threads limit: %v", err) } @@ -859,7 +879,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S logrus.Errorf(err.Error()) } - daemonRepo := filepath.Join(config.Root, "containers") + daemonRepo := filepath.Join(configStore.Root, "containers") if err := idtools.MkdirAllAndChown(daemonRepo, 0o710, idtools.Identity{ UID: idtools.CurrentIdentity().UID, GID: rootIDs.GID, @@ -867,20 +887,11 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S return nil, err } - // Create the directory where we'll store the runtime scripts (i.e. in - // order to support runtimeArgs) - if err = os.Mkdir(filepath.Join(config.Root, "runtimes"), 0o700); err != nil && !errors.Is(err, os.ErrExist) { - return nil, err - } - if err := d.loadRuntimes(); err != nil { - return nil, err - } - if isWindows { // Note that permissions (0o700) are ignored on Windows; passing them to // show intent only. We could consider using idtools.MkdirAndChown here // to apply an ACL. - if err = os.Mkdir(filepath.Join(config.Root, "credentialspecs"), 0o700); err != nil && !errors.Is(err, os.ErrExist) { + if err = os.Mkdir(filepath.Join(configStore.Root, "credentialspecs"), 0o700); err != nil && !errors.Is(err, os.ErrExist) { return nil, err } } @@ -888,7 +899,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S d.registryService = registryService dlogger.RegisterPluginGetter(d.PluginStore) - metricsSockPath, err := d.listenMetricsSock(config) + metricsSockPath, err := d.listenMetricsSock(&configStore.Config) if err != nil { return nil, err } @@ -927,20 +938,20 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), } - if config.ContainerdAddr != "" { - d.containerdCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(config.ContainerdNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second)) + if configStore.ContainerdAddr != "" { + d.containerdCli, err = containerd.New(configStore.ContainerdAddr, containerd.WithDefaultNamespace(configStore.ContainerdNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second)) if err != nil { - return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr) + return nil, errors.Wrapf(err, "failed to dial %q", configStore.ContainerdAddr) } } createPluginExec := func(m *plugin.Manager) (plugin.Executor, error) { var pluginCli *containerd.Client - if config.ContainerdAddr != "" { - pluginCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(config.ContainerdPluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second)) + if configStore.ContainerdAddr != "" { + pluginCli, err = containerd.New(configStore.ContainerdAddr, containerd.WithDefaultNamespace(configStore.ContainerdPluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second)) if err != nil { - return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr) + return nil, errors.Wrapf(err, "failed to dial %q", configStore.ContainerdAddr) } } @@ -949,22 +960,22 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S shimOpts interface{} ) if runtime.GOOS != "windows" { - shim, shimOpts, err = d.getRuntime(config, config.DefaultRuntime) + shim, shimOpts, err = runtimes.Get(configStore.DefaultRuntime) if err != nil { return nil, err } } - return pluginexec.New(ctx, getPluginExecRoot(config), pluginCli, config.ContainerdPluginNamespace, m, shim, shimOpts) + return pluginexec.New(ctx, getPluginExecRoot(&configStore.Config), pluginCli, configStore.ContainerdPluginNamespace, m, shim, shimOpts) } // Plugin system initialization should happen before restore. Do not change order. d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{ - Root: filepath.Join(config.Root, "plugins"), - ExecRoot: getPluginExecRoot(config), + Root: filepath.Join(configStore.Root, "plugins"), + ExecRoot: getPluginExecRoot(&configStore.Config), Store: d.PluginStore, CreateExecutor: createPluginExec, RegistryService: registryService, - LiveRestoreEnabled: config.LiveRestoreEnabled, + LiveRestoreEnabled: configStore.LiveRestoreEnabled, LogPluginEvent: d.LogPluginEvent, // todo: make private AuthzMiddleware: authzMiddleware, }) @@ -972,13 +983,13 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S return nil, errors.Wrap(err, "couldn't create plugin manager") } - d.defaultLogConfig, err = defaultLogConfig(config) + d.defaultLogConfig, err = defaultLogConfig(&configStore.Config) if err != nil { return nil, errors.Wrap(err, "failed to set log opts") } logrus.Debugf("Using default logging driver %s", d.defaultLogConfig.Type) - d.volumes, err = volumesservice.NewVolumeService(config.Root, d.PluginStore, rootIDs, d) + d.volumes, err = volumesservice.NewVolumeService(configStore.Root, d.PluginStore, rootIDs, d) if err != nil { return nil, err } @@ -991,11 +1002,11 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S // at this point. // // TODO(thaJeztah) add a utility to only collect the CgroupDevicesEnabled information - if runtime.GOOS == "linux" && !userns.RunningInUserNS() && !getSysInfo(config).CgroupDevicesEnabled { + if runtime.GOOS == "linux" && !userns.RunningInUserNS() && !getSysInfo(&configStore.Config).CgroupDevicesEnabled { return nil, errors.New("Devices cgroup isn't mounted") } - d.id, err = loadOrCreateID(filepath.Join(config.Root, "engine-id")) + d.id, err = loadOrCreateID(filepath.Join(configStore.Root, "engine-id")) if err != nil { return nil, err } @@ -1008,7 +1019,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S d.statsCollector = d.newStatsCollector(1 * time.Second) d.EventsService = events.New() - d.root = config.Root + d.root = configStore.Root d.idMapping = idMapping d.linkIndex = newLinkIndex() @@ -1023,7 +1034,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } else if driverName != "" { logrus.Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", driverName) } else { - driverName = config.GraphDriver + driverName = configStore.GraphDriver } if d.UsesSnapshotter() { @@ -1039,7 +1050,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S // Configure and validate the kernels security support. Note this is a Linux/FreeBSD // operation only, so it is safe to pass *just* the runtime OS graphdriver. - if err := configureKernelSecuritySupport(config, driverName); err != nil { + if err := configureKernelSecuritySupport(&configStore.Config, driverName); err != nil { return nil, err } d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{ @@ -1052,13 +1063,13 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S }) } else { layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{ - Root: config.Root, - MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"), + Root: configStore.Root, + MetadataStorePathTemplate: filepath.Join(configStore.Root, "image", "%s", "layerdb"), GraphDriver: driverName, - GraphDriverOptions: config.GraphOptions, + GraphDriverOptions: configStore.GraphOptions, IDMapping: idMapping, PluginGetter: d.PluginStore, - ExperimentalEnabled: config.Experimental, + ExperimentalEnabled: configStore.Experimental, }) if err != nil { return nil, err @@ -1066,11 +1077,11 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S // Configure and validate the kernels security support. Note this is a Linux/FreeBSD // operation only, so it is safe to pass *just* the runtime OS graphdriver. - if err := configureKernelSecuritySupport(config, layerStore.DriverName()); err != nil { + if err := configureKernelSecuritySupport(&configStore.Config, layerStore.DriverName()); err != nil { return nil, err } - imageRoot := filepath.Join(config.Root, "image", layerStore.DriverName()) + imageRoot := filepath.Join(configStore.Root, "image", layerStore.DriverName()) ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb")) if err != nil { return nil, err @@ -1144,11 +1155,11 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S go d.execCommandGC() - if err := d.initLibcontainerd(ctx, config); err != nil { + if err := d.initLibcontainerd(ctx, &configStore.Config); err != nil { return nil, err } - if err := d.restore(config); err != nil { + if err := d.restore(configStore); err != nil { return nil, err } close(d.startupDone) @@ -1210,7 +1221,7 @@ func (daemon *Daemon) shutdownContainer(c *container.Container) error { // A negative (-1) timeout means "indefinitely", which means that containers // are not forcibly killed, and the daemon shuts down after all containers exit. func (daemon *Daemon) ShutdownTimeout() int { - return daemon.shutdownTimeout(daemon.config()) + return daemon.shutdownTimeout(&daemon.config().Config) } func (daemon *Daemon) shutdownTimeout(cfg *config.Config) int { @@ -1241,7 +1252,7 @@ func (daemon *Daemon) Shutdown(ctx context.Context) error { // Keep mounts and networking running on daemon shutdown if // we are to keep containers running and restore them. - cfg := daemon.config() + cfg := &daemon.config().Config if cfg.LiveRestoreEnabled && daemon.containers != nil { // check if there are any running containers, if none we should do some cleanup if ls, err := daemon.Containers(ctx, &types.ContainerListOptions{}); len(ls) != 0 || err != nil { @@ -1523,7 +1534,7 @@ func (daemon *Daemon) RawSysInfo() *sysinfo.SysInfo { // We check if sysInfo is not set here, to allow some test to // override the actual sysInfo. if daemon.sysInfo == nil { - daemon.sysInfo = getSysInfo(daemon.config()) + daemon.sysInfo = getSysInfo(&daemon.config().Config) } }) diff --git a/daemon/daemon_linux.go b/daemon/daemon_linux.go index 83a4fa2cae..a3d39fc83c 100644 --- a/daemon/daemon_linux.go +++ b/daemon/daemon_linux.go @@ -239,18 +239,18 @@ func kernelSupportsRecursivelyReadOnly() error { return kernelSupportsRROErr } -func supportsRecursivelyReadOnly(cfg *config.Config, runtime string) error { +func supportsRecursivelyReadOnly(cfg *configStore, runtime string) error { if err := kernelSupportsRecursivelyReadOnly(); err != nil { return fmt.Errorf("rro is not supported: %w (kernel is older than 5.12?)", err) } if runtime == "" { runtime = cfg.DefaultRuntime } - rt := cfg.GetRuntime(runtime) - if rt.Features == nil { + features := cfg.Runtimes.Features(runtime) + if features == nil { return fmt.Errorf("rro is not supported by runtime %q: OCI features struct is not available", runtime) } - for _, s := range rt.Features.MountOptions { + for _, s := range features.MountOptions { if s == "rro" { return nil } diff --git a/daemon/daemon_linux_test.go b/daemon/daemon_linux_test.go index cae47037c4..5bc24f45a7 100644 --- a/daemon/daemon_linux_test.go +++ b/daemon/daemon_linux_test.go @@ -10,7 +10,6 @@ import ( "testing" containertypes "github.com/docker/docker/api/types/container" - "github.com/docker/docker/daemon/config" "github.com/docker/docker/libnetwork/testutils" "github.com/docker/docker/libnetwork/types" "github.com/google/go-cmp/cmp/cmpopts" @@ -178,7 +177,7 @@ func TestNotCleanupMounts(t *testing.T) { func TestValidateContainerIsolationLinux(t *testing.T) { d := Daemon{} - _, err := d.verifyContainerSettings(&config.Config{}, &containertypes.HostConfig{Isolation: containertypes.IsolationHyperV}, nil, false) + _, err := d.verifyContainerSettings(&configStore{}, &containertypes.HostConfig{Isolation: containertypes.IsolationHyperV}, nil, false) assert.Check(t, is.Error(err, "invalid isolation 'hyperv' on linux")) } @@ -250,7 +249,7 @@ func TestRootMountCleanup(t *testing.T) { testRoot, err := os.MkdirTemp("", t.Name()) assert.NilError(t, err) defer os.RemoveAll(testRoot) - cfg := &config.Config{} + cfg := &configStore{} err = mount.MakePrivate(testRoot) assert.NilError(t, err) @@ -266,16 +265,16 @@ func TestRootMountCleanup(t *testing.T) { d := &Daemon{root: cfg.Root} d.configStore.Store(cfg) - unmountFile := getUnmountOnShutdownPath(cfg) + unmountFile := getUnmountOnShutdownPath(&cfg.Config) t.Run("regular dir no mountpoint", func(t *testing.T) { - err = setupDaemonRootPropagation(cfg) + err = setupDaemonRootPropagation(&cfg.Config) assert.NilError(t, err) _, err = os.Stat(unmountFile) assert.NilError(t, err) checkMounted(t, cfg.Root, true) - assert.Assert(t, d.cleanupMounts(cfg)) + assert.Assert(t, d.cleanupMounts(&cfg.Config)) checkMounted(t, cfg.Root, false) _, err = os.Stat(unmountFile) @@ -287,13 +286,13 @@ func TestRootMountCleanup(t *testing.T) { assert.NilError(t, err) defer mount.Unmount(cfg.Root) - err = setupDaemonRootPropagation(cfg) + err = setupDaemonRootPropagation(&cfg.Config) assert.NilError(t, err) assert.Check(t, ensureShared(cfg.Root)) _, err = os.Stat(unmountFile) assert.Assert(t, os.IsNotExist(err)) - assert.Assert(t, d.cleanupMounts(cfg)) + assert.Assert(t, d.cleanupMounts(&cfg.Config)) checkMounted(t, cfg.Root, true) }) @@ -303,14 +302,14 @@ func TestRootMountCleanup(t *testing.T) { assert.NilError(t, err) defer mount.Unmount(cfg.Root) - err = setupDaemonRootPropagation(cfg) + err = setupDaemonRootPropagation(&cfg.Config) assert.NilError(t, err) if _, err := os.Stat(unmountFile); err == nil { t.Fatal("unmount file should not exist") } - assert.Assert(t, d.cleanupMounts(cfg)) + assert.Assert(t, d.cleanupMounts(&cfg.Config)) checkMounted(t, cfg.Root, true) assert.Assert(t, mount.Unmount(cfg.Root)) }) @@ -323,13 +322,13 @@ func TestRootMountCleanup(t *testing.T) { err = os.WriteFile(unmountFile, nil, 0644) assert.NilError(t, err) - err = setupDaemonRootPropagation(cfg) + err = setupDaemonRootPropagation(&cfg.Config) assert.NilError(t, err) _, err = os.Stat(unmountFile) assert.Check(t, os.IsNotExist(err), err) checkMounted(t, cfg.Root, false) - assert.Assert(t, d.cleanupMounts(cfg)) + assert.Assert(t, d.cleanupMounts(&cfg.Config)) }) } diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go index 34e871e669..ced1ecfab9 100644 --- a/daemon/daemon_test.go +++ b/daemon/daemon_test.go @@ -8,7 +8,6 @@ import ( containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" - "github.com/docker/docker/daemon/config" "github.com/docker/docker/errdefs" "github.com/docker/docker/libnetwork" "github.com/docker/docker/pkg/idtools" @@ -301,7 +300,7 @@ func TestMerge(t *testing.T) { func TestValidateContainerIsolation(t *testing.T) { d := Daemon{} - _, err := d.verifyContainerSettings(&config.Config{}, &containertypes.HostConfig{Isolation: containertypes.Isolation("invalid")}, nil, false) + _, err := d.verifyContainerSettings(&configStore{}, &containertypes.HostConfig{Isolation: containertypes.Isolation("invalid")}, nil, false) assert.Check(t, is.Error(err, "invalid isolation 'invalid' on "+runtime.GOOS)) } diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index 3d7e8b4586..447407f0f6 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -639,7 +639,7 @@ func isRunningSystemd() bool { // verifyPlatformContainerSettings performs platform-specific validation of the // hostconfig and config structures. -func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *config.Config, hostConfig *containertypes.HostConfig, update bool) (warnings []string, err error) { +func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *configStore, hostConfig *containertypes.HostConfig, update bool) (warnings []string, err error) { if hostConfig == nil { return nil, nil } @@ -691,7 +691,7 @@ func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *config.Config, h return warnings, fmt.Errorf("cannot share the host PID namespace when user namespaces are enabled") } } - if hostConfig.CgroupParent != "" && UsingSystemd(daemonCfg) { + if hostConfig.CgroupParent != "" && UsingSystemd(&daemonCfg.Config) { // CgroupParent for systemd cgroup should be named as "xxx.slice" if len(hostConfig.CgroupParent) <= 6 || !strings.HasSuffix(hostConfig.CgroupParent, ".slice") { return warnings, fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"") @@ -701,7 +701,7 @@ func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *config.Config, h hostConfig.Runtime = daemonCfg.DefaultRuntime } - if _, _, err := daemon.getRuntime(daemonCfg, hostConfig.Runtime); err != nil { + if _, _, err := daemonCfg.Runtimes.Get(hostConfig.Runtime); err != nil { return warnings, err } @@ -757,7 +757,7 @@ func verifyDaemonSettings(conf *config.Config) error { configureRuntimes(conf) if rtName := conf.DefaultRuntime; rtName != "" { - if conf.GetRuntime(rtName) == nil { + if _, ok := conf.Runtimes[rtName]; !ok { if !config.IsPermissibleC8dRuntimeName(rtName) { return fmt.Errorf("specified default runtime '%s' does not exist", rtName) } diff --git a/daemon/daemon_unix_test.go b/daemon/daemon_unix_test.go index 83c307c9bf..a64bc9406d 100644 --- a/daemon/daemon_unix_test.go +++ b/daemon/daemon_unix_test.go @@ -245,7 +245,7 @@ func TestParseSecurityOpt(t *testing.T) { } func TestParseNNPSecurityOptions(t *testing.T) { - daemonCfg := &config.Config{NoNewPrivileges: true} + daemonCfg := &configStore{Config: config.Config{NoNewPrivileges: true}} daemon := &Daemon{} daemon.configStore.Store(daemonCfg) opts := &container.SecurityOptions{} @@ -254,7 +254,7 @@ func TestParseNNPSecurityOptions(t *testing.T) { // test NNP when "daemon:true" and "no-new-privileges=false"" cfg.SecurityOpt = []string{"no-new-privileges=false"} - if err := daemon.parseSecurityOpt(daemonCfg, opts, cfg); err != nil { + if err := daemon.parseSecurityOpt(&daemonCfg.Config, opts, cfg); err != nil { t.Fatalf("Unexpected daemon.parseSecurityOpt error: %v", err) } if opts.NoNewPrivileges { @@ -265,7 +265,7 @@ func TestParseNNPSecurityOptions(t *testing.T) { daemonCfg.NoNewPrivileges = false cfg.SecurityOpt = []string{"no-new-privileges=true"} - if err := daemon.parseSecurityOpt(daemonCfg, opts, cfg); err != nil { + if err := daemon.parseSecurityOpt(&daemonCfg.Config, opts, cfg); err != nil { t.Fatalf("Unexpected daemon.parseSecurityOpt error: %v", err) } if !opts.NoNewPrivileges { diff --git a/daemon/daemon_windows.go b/daemon/daemon_windows.go index f906bba554..40e73d13b1 100644 --- a/daemon/daemon_windows.go +++ b/daemon/daemon_windows.go @@ -171,7 +171,7 @@ func verifyPlatformContainerResources(resources *containertypes.Resources, isHyp // verifyPlatformContainerSettings performs platform-specific validation of the // hostconfig and config structures. -func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *config.Config, hostConfig *containertypes.HostConfig, update bool) (warnings []string, err error) { +func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *configStore, hostConfig *containertypes.HostConfig, update bool) (warnings []string, err error) { if hostConfig == nil { return nil, nil } @@ -556,10 +556,6 @@ func (daemon *Daemon) setupSeccompProfile(*config.Config) error { return nil } -func (daemon *Daemon) loadRuntimes() error { - return nil -} - func setupResolvConf(config *config.Config) {} func getSysInfo(*config.Config) *sysinfo.SysInfo { diff --git a/daemon/delete.go b/daemon/delete.go index d9192e70fa..6bb7fdf03e 100644 --- a/daemon/delete.go +++ b/daemon/delete.go @@ -25,7 +25,7 @@ import ( // fails. If the remove succeeds, the container name is released, and // network links are removed. func (daemon *Daemon) ContainerRm(name string, config *types.ContainerRmConfig) error { - return daemon.containerRm(daemon.config(), name, config) + return daemon.containerRm(&daemon.config().Config, name, config) } func (daemon *Daemon) containerRm(cfg *config.Config, name string, opts *types.ContainerRmConfig) error { diff --git a/daemon/exec.go b/daemon/exec.go index 68e27c0cf6..f458ae7f50 100644 --- a/daemon/exec.go +++ b/daemon/exec.go @@ -252,7 +252,7 @@ func (daemon *Daemon) ContainerExecStart(ctx context.Context, name string, optio p.Cwd = "/" } - daemonCfg := daemon.config() + daemonCfg := &daemon.config().Config if err := daemon.execSetPlatformOpt(ctx, daemonCfg, ec, p); err != nil { return err } diff --git a/daemon/exec_linux_test.go b/daemon/exec_linux_test.go index 409eb0cae5..0207c3df4f 100644 --- a/daemon/exec_linux_test.go +++ b/daemon/exec_linux_test.go @@ -9,7 +9,6 @@ import ( "github.com/containerd/containerd/pkg/apparmor" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" - "github.com/docker/docker/daemon/config" specs "github.com/opencontainers/runtime-spec/specs-go" "gotest.tools/v3/assert" ) @@ -50,7 +49,7 @@ func TestExecSetPlatformOptAppArmor(t *testing.T) { }, } - cfg := &config.Config{} + cfg := &configStore{} d := &Daemon{} d.configStore.Store(cfg) @@ -83,7 +82,7 @@ func TestExecSetPlatformOptAppArmor(t *testing.T) { ec := &container.ExecConfig{Container: c, Privileged: execPrivileged} p := &specs.Process{} - err := d.execSetPlatformOpt(context.Background(), cfg, ec, p) + err := d.execSetPlatformOpt(context.Background(), &cfg.Config, ec, p) assert.NilError(t, err) assert.Equal(t, p.ApparmorProfile, tc.expectedProfile) }) diff --git a/daemon/info.go b/daemon/info.go index 96889f4ddd..241a8ebb9c 100644 --- a/daemon/info.go +++ b/daemon/info.go @@ -64,14 +64,14 @@ func (daemon *Daemon) SystemInfo() *types.Info { daemon.fillContainerStates(v) daemon.fillDebugInfo(v) - daemon.fillAPIInfo(v, cfg) + daemon.fillAPIInfo(v, &cfg.Config) // Retrieve platform specific info - daemon.fillPlatformInfo(v, sysInfo, cfg) + daemon.fillPlatformInfo(v, sysInfo, &cfg.Config) daemon.fillDriverInfo(v) - daemon.fillPluginsInfo(v, cfg) - daemon.fillSecurityOptions(v, sysInfo, cfg) + daemon.fillPluginsInfo(v, &cfg.Config) + daemon.fillSecurityOptions(v, sysInfo, &cfg.Config) daemon.fillLicense(v) - daemon.fillDefaultAddressPools(v, cfg) + daemon.fillDefaultAddressPools(v, &cfg.Config) return v } @@ -117,7 +117,7 @@ func (daemon *Daemon) SystemVersion() types.Version { v.Platform.Name = dockerversion.PlatformName - daemon.fillPlatformVersion(&v, cfg) + daemon.fillPlatformVersion(&v, &cfg.Config) return v } diff --git a/daemon/info_unix.go b/daemon/info_unix.go index 4afcca7ee3..22d313fb04 100644 --- a/daemon/info_unix.go +++ b/daemon/info_unix.go @@ -38,22 +38,22 @@ func (daemon *Daemon) fillPlatformInfo(v *types.Info, sysInfo *sysinfo.SysInfo, v.CPUSet = sysInfo.Cpuset v.PidsLimit = sysInfo.PidsLimit } - v.Runtimes = cfg.GetAllRuntimes() + v.Runtimes = make(map[string]types.Runtime) + for n, r := range cfg.Runtimes { + v.Runtimes[n] = types.Runtime{ + Path: r.Path, + Args: append([]string(nil), r.Args...), + } + } v.DefaultRuntime = cfg.DefaultRuntime v.RuncCommit.ID = "N/A" v.ContainerdCommit.ID = "N/A" v.InitCommit.ID = "N/A" - if rt := cfg.GetRuntime(v.DefaultRuntime); rt != nil { - if rv, err := exec.Command(rt.Path, "--version").Output(); err == nil { - if _, _, commit, err := parseRuntimeVersion(string(rv)); err != nil { - logrus.Warnf("failed to parse %s version: %v", rt.Path, err) - } else { - v.RuncCommit.ID = commit - } - } else { - logrus.Warnf("failed to retrieve %s version: %v", rt.Path, err) - } + if _, _, commit, err := parseDefaultRuntimeVersion(cfg); err != nil { + logrus.Warnf(err.Error()) + } else { + v.RuncCommit.ID = commit } if rv, err := daemon.containerd.Version(context.Background()); err == nil { @@ -177,23 +177,16 @@ func (daemon *Daemon) fillPlatformVersion(v *types.Version, cfg *config.Config) }) } - defaultRuntime := cfg.DefaultRuntime - if rt := cfg.GetRuntime(defaultRuntime); rt != nil { - if rv, err := exec.Command(rt.Path, "--version").Output(); err == nil { - if _, ver, commit, err := parseRuntimeVersion(string(rv)); err != nil { - logrus.Warnf("failed to parse %s version: %v", rt.Path, err) - } else { - v.Components = append(v.Components, types.ComponentVersion{ - Name: defaultRuntime, - Version: ver, - Details: map[string]string{ - "GitCommit": commit, - }, - }) - } - } else { - logrus.Warnf("failed to retrieve %s version: %v", rt.Path, err) - } + if _, ver, commit, err := parseDefaultRuntimeVersion(cfg); err != nil { + logrus.Warnf(err.Error()) + } else { + v.Components = append(v.Components, types.ComponentVersion{ + Name: cfg.DefaultRuntime, + Version: ver, + Details: map[string]string{ + "GitCommit": commit, + }, + }) } if initBinary, err := cfg.LookupInitPath(); err != nil { @@ -318,7 +311,7 @@ func parseInitVersion(v string) (version string, commit string, err error) { // runc version 1.0.0-rc5+dev // commit: 69663f0bd4b60df09991c08812a60108003fa340 // spec: 1.0.0 -func parseRuntimeVersion(v string) (runtime string, version string, commit string, err error) { +func parseRuntimeVersion(v string) (runtime, version, commit string, err error) { lines := strings.Split(strings.TrimSpace(v), "\n") for _, line := range lines { if strings.Contains(line, "version") { @@ -338,6 +331,21 @@ func parseRuntimeVersion(v string) (runtime string, version string, commit strin return runtime, version, commit, err } +func parseDefaultRuntimeVersion(cfg *config.Config) (runtime, version, commit string, err error) { + if rt, ok := cfg.Runtimes[cfg.DefaultRuntime]; ok { + rv, err := exec.Command(rt.Path, "--version").Output() + if err != nil { + return "", "", "", fmt.Errorf("failed to retrieve %s version: %w", rt.Path, err) + } + runtime, version, commit, err := parseRuntimeVersion(string(rv)) + if err != nil { + return "", "", "", fmt.Errorf("failed to parse %s version: %w", rt.Path, err) + } + return runtime, version, commit, err + } + return "", "", "", nil +} + func cgroupNamespacesEnabled(sysInfo *sysinfo.SysInfo, cfg *config.Config) bool { return sysInfo.CgroupNamespaces && containertypes.CgroupnsMode(cfg.CgroupNamespaceMode).IsPrivate() } diff --git a/daemon/inspect.go b/daemon/inspect.go index 677c223996..32ec615016 100644 --- a/daemon/inspect.go +++ b/daemon/inspect.go @@ -41,7 +41,7 @@ func (daemon *Daemon) ContainerInspectCurrent(ctx context.Context, name string, ctr.Lock() - base, err := daemon.getInspectData(daemon.config(), ctr) + base, err := daemon.getInspectData(&daemon.config().Config, ctr) if err != nil { ctr.Unlock() return nil, err @@ -106,7 +106,7 @@ func (daemon *Daemon) containerInspect120(name string) (*v1p20.ContainerJSON, er ctr.Lock() defer ctr.Unlock() - base, err := daemon.getInspectData(daemon.config(), ctr) + base, err := daemon.getInspectData(&daemon.config().Config, ctr) if err != nil { return nil, err } diff --git a/daemon/inspect_linux.go b/daemon/inspect_linux.go index 409896c43a..77cb008677 100644 --- a/daemon/inspect_linux.go +++ b/daemon/inspect_linux.go @@ -29,7 +29,7 @@ func (daemon *Daemon) containerInspectPre120(ctx context.Context, name string) ( ctr.Lock() defer ctr.Unlock() - base, err := daemon.getInspectData(daemon.config(), ctr) + base, err := daemon.getInspectData(&daemon.config().Config, ctr) if err != nil { return nil, err } diff --git a/daemon/inspect_test.go b/daemon/inspect_test.go index a68ab83d33..8faab2805e 100644 --- a/daemon/inspect_test.go +++ b/daemon/inspect_test.go @@ -5,7 +5,6 @@ import ( containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" - "github.com/docker/docker/daemon/config" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" ) @@ -24,13 +23,13 @@ func TestGetInspectData(t *testing.T) { if d.UsesSnapshotter() { t.Skip("does not apply to containerd snapshotters, which don't have RWLayer set") } - cfg := &config.Config{} + cfg := &configStore{} d.configStore.Store(cfg) - _, err := d.getInspectData(cfg, c) + _, err := d.getInspectData(&cfg.Config, c) assert.Check(t, is.ErrorContains(err, "RWLayer of container inspect-me is unexpectedly nil")) c.Dead = true - _, err = d.getInspectData(cfg, c) + _, err = d.getInspectData(&cfg.Config, c) assert.Check(t, err) } diff --git a/daemon/monitor.go b/daemon/monitor.go index f298c8fbba..6dd5dbab15 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -102,7 +102,7 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine } else { c.SetStopped(&exitStatus) if !c.HasBeenManuallyRestarted { - defer daemon.autoRemove(cfg, c) + defer daemon.autoRemove(&cfg.Config, c) } } defer c.Unlock() // needs to be called before autoRemove @@ -131,7 +131,7 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine daemon.setStateCounter(c) c.CheckpointTo(daemon.containersReplica) c.Unlock() - defer daemon.autoRemove(cfg, c) + defer daemon.autoRemove(&cfg.Config, c) if err != restartmanager.ErrRestartCanceled { logrus.Errorf("restartmanger wait error: %+v", err) } diff --git a/daemon/network.go b/daemon/network.go index cb3693d3e5..5a52fb4616 100644 --- a/daemon/network.go +++ b/daemon/network.go @@ -161,7 +161,7 @@ func (daemon *Daemon) startIngressWorker() { select { case r := <-ingressJobsChannel: if r.create != nil { - daemon.setupIngress(daemon.config(), r.create, r.ip, ingressID) + daemon.setupIngress(&daemon.config().Config, r.create, r.ip, ingressID) ingressID = r.create.ID } else { daemon.releaseIngress(ingressID) @@ -278,13 +278,13 @@ func (daemon *Daemon) WaitForDetachment(ctx context.Context, networkName, networ // CreateManagedNetwork creates an agent network. func (daemon *Daemon) CreateManagedNetwork(create clustertypes.NetworkCreateRequest) error { - _, err := daemon.createNetwork(daemon.config(), create.NetworkCreateRequest, create.ID, true) + _, err := daemon.createNetwork(&daemon.config().Config, create.NetworkCreateRequest, create.ID, true) return err } // CreateNetwork creates a network with the given name, driver and other optional parameters func (daemon *Daemon) CreateNetwork(create types.NetworkCreateRequest) (*types.NetworkCreateResponse, error) { - return daemon.createNetwork(daemon.config(), create, "", false) + return daemon.createNetwork(&daemon.config().Config, create, "", false) } func (daemon *Daemon) createNetwork(cfg *config.Config, create types.NetworkCreateRequest, id string, agent bool) (*types.NetworkCreateResponse, error) { diff --git a/daemon/oci_linux.go b/daemon/oci_linux.go index 90f5beee64..e336d2c4df 100644 --- a/daemon/oci_linux.go +++ b/daemon/oci_linux.go @@ -489,7 +489,7 @@ func inSlice(slice []string, s string) bool { } // withMounts sets the container's mounts -func withMounts(daemon *Daemon, daemonCfg *dconfig.Config, c *container.Container) coci.SpecOpts { +func withMounts(daemon *Daemon, daemonCfg *configStore, c *container.Container) coci.SpecOpts { return func(ctx context.Context, _ coci.Client, _ *containers.Container, s *coci.Spec) (err error) { if err := daemon.setupContainerMountsRoot(c); err != nil { return err @@ -1019,23 +1019,23 @@ func WithUser(c *container.Container) coci.SpecOpts { } } -func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *dconfig.Config, c *container.Container) (retSpec *specs.Spec, err error) { +func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *configStore, c *container.Container) (retSpec *specs.Spec, err error) { var ( opts []coci.SpecOpts s = oci.DefaultSpec() ) opts = append(opts, - withCommonOptions(daemon, daemonCfg, c), - withCgroups(daemon, daemonCfg, c), + withCommonOptions(daemon, &daemonCfg.Config, c), + withCgroups(daemon, &daemonCfg.Config, c), WithResources(c), WithSysctls(c), WithDevices(daemon, c), - withRlimits(daemon, daemonCfg, c), + withRlimits(daemon, &daemonCfg.Config, c), WithNamespaces(daemon, c), WithCapabilities(c), WithSeccomp(daemon, c), withMounts(daemon, daemonCfg, c), - withLibnetwork(daemon, daemonCfg, c), + withLibnetwork(daemon, &daemonCfg.Config, c), WithApparmor(c), WithSelinux(c), WithOOMScore(&c.HostConfig.OomScoreAdj), @@ -1069,7 +1069,7 @@ func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *dconfig.Config, opts = append(opts, coci.WithReadonlyPaths(c.HostConfig.ReadonlyPaths)) } if daemonCfg.Rootless { - opts = append(opts, withRootless(daemon, daemonCfg)) + opts = append(opts, withRootless(daemon, &daemonCfg.Config)) } var snapshotter, snapshotKey string diff --git a/daemon/oci_linux_test.go b/daemon/oci_linux_test.go index b3e808e60b..072c0bfba1 100644 --- a/daemon/oci_linux_test.go +++ b/daemon/oci_linux_test.go @@ -82,7 +82,7 @@ func TestTmpfsDevShmNoDupMount(t *testing.T) { d := setupFakeDaemon(t, c) defer cleanupFakeContainer(c) - _, err := d.createSpec(context.TODO(), &config.Config{}, c) + _, err := d.createSpec(context.TODO(), &configStore{}, c) assert.Check(t, err) } @@ -101,7 +101,7 @@ func TestIpcPrivateVsReadonly(t *testing.T) { d := setupFakeDaemon(t, c) defer cleanupFakeContainer(c) - s, err := d.createSpec(context.TODO(), &config.Config{}, c) + s, err := d.createSpec(context.TODO(), &configStore{}, c) assert.Check(t, err) // Find the /dev/shm mount in ms, check it does not have ro @@ -131,7 +131,7 @@ func TestSysctlOverride(t *testing.T) { defer cleanupFakeContainer(c) // Ensure that the implicit sysctl is set correctly. - s, err := d.createSpec(context.TODO(), &config.Config{}, c) + s, err := d.createSpec(context.TODO(), &configStore{}, c) assert.NilError(t, err) assert.Equal(t, s.Hostname, "foobar") assert.Equal(t, s.Linux.Sysctl["kernel.domainname"], c.Config.Domainname) @@ -147,14 +147,14 @@ func TestSysctlOverride(t *testing.T) { assert.Assert(t, c.HostConfig.Sysctls["kernel.domainname"] != c.Config.Domainname) c.HostConfig.Sysctls["net.ipv4.ip_unprivileged_port_start"] = "1024" - s, err = d.createSpec(context.TODO(), &config.Config{}, c) + s, err = d.createSpec(context.TODO(), &configStore{}, c) assert.NilError(t, err) assert.Equal(t, s.Hostname, "foobar") assert.Equal(t, s.Linux.Sysctl["kernel.domainname"], c.HostConfig.Sysctls["kernel.domainname"]) assert.Equal(t, s.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], c.HostConfig.Sysctls["net.ipv4.ip_unprivileged_port_start"]) // Ensure the ping_group_range is not set on a daemon with user-namespaces enabled - s, err = d.createSpec(context.TODO(), &config.Config{RemappedRoot: "dummy:dummy"}, c) + s, err = d.createSpec(context.TODO(), &configStore{Config: config.Config{RemappedRoot: "dummy:dummy"}}, c) assert.NilError(t, err) _, ok := s.Linux.Sysctl["net.ipv4.ping_group_range"] assert.Assert(t, !ok) @@ -162,7 +162,7 @@ func TestSysctlOverride(t *testing.T) { // Ensure the ping_group_range is set on a container in "host" userns mode // on a daemon with user-namespaces enabled c.HostConfig.UsernsMode = "host" - s, err = d.createSpec(context.TODO(), &config.Config{RemappedRoot: "dummy:dummy"}, c) + s, err = d.createSpec(context.TODO(), &configStore{Config: config.Config{RemappedRoot: "dummy:dummy"}}, c) assert.NilError(t, err) assert.Equal(t, s.Linux.Sysctl["net.ipv4.ping_group_range"], "0 2147483647") } @@ -182,7 +182,7 @@ func TestSysctlOverrideHost(t *testing.T) { defer cleanupFakeContainer(c) // Ensure that the implicit sysctl is not set - s, err := d.createSpec(context.TODO(), &config.Config{}, c) + s, err := d.createSpec(context.TODO(), &configStore{}, c) assert.NilError(t, err) assert.Equal(t, s.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], "") assert.Equal(t, s.Linux.Sysctl["net.ipv4.ping_group_range"], "") @@ -190,7 +190,7 @@ func TestSysctlOverrideHost(t *testing.T) { // Set an explicit sysctl. c.HostConfig.Sysctls["net.ipv4.ip_unprivileged_port_start"] = "1024" - s, err = d.createSpec(context.TODO(), &config.Config{}, c) + s, err = d.createSpec(context.TODO(), &configStore{}, c) assert.NilError(t, err) assert.Equal(t, s.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], c.HostConfig.Sysctls["net.ipv4.ip_unprivileged_port_start"]) } diff --git a/daemon/oci_windows.go b/daemon/oci_windows.go index 6dbe475254..7809a02c53 100644 --- a/daemon/oci_windows.go +++ b/daemon/oci_windows.go @@ -28,7 +28,7 @@ const ( credentialSpecFileLocation = "CredentialSpecs" ) -func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *config.Config, c *container.Container) (*specs.Spec, error) { +func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *configStore, c *container.Container) (*specs.Spec, error) { img, err := daemon.imageService.GetImage(ctx, string(c.ImageID), imagetypes.GetImageOpts{}) if err != nil { return nil, err @@ -143,7 +143,7 @@ func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *config.Config, return nil, errors.Wrapf(err, "container %s", c.ID) } - dnsSearch := daemon.getDNSSearchSettings(daemonCfg, c) + dnsSearch := daemon.getDNSSearchSettings(&daemonCfg.Config, c) // Get endpoints for the libnetwork allocated networks to the container var epList []string diff --git a/daemon/prune.go b/daemon/prune.go index 6f3139b832..d2b265a678 100644 --- a/daemon/prune.go +++ b/daemon/prune.go @@ -56,7 +56,7 @@ func (daemon *Daemon) ContainersPrune(ctx context.Context, pruneFilters filters. return nil, err } - cfg := daemon.config() + cfg := &daemon.config().Config allContainers := daemon.List() for _, c := range allContainers { select { diff --git a/daemon/reload.go b/daemon/reload.go index 7862117730..682c71786c 100644 --- a/daemon/reload.go +++ b/daemon/reload.go @@ -72,11 +72,13 @@ func (tx *reloadTxn) Rollback() error { func (daemon *Daemon) Reload(conf *config.Config) error { daemon.configReload.Lock() defer daemon.configReload.Unlock() - copied, err := copystructure.Copy(daemon.config()) + copied, err := copystructure.Copy(daemon.config().Config) if err != nil { return err } - newCfg := copied.(*config.Config) + newCfg := &configStore{ + Config: copied.(config.Config), + } attributes := map[string]string{} @@ -91,7 +93,7 @@ func (daemon *Daemon) Reload(conf *config.Config) error { // executing any registered rollback functions. var txn reloadTxn - for _, reload := range []func(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error{ + for _, reload := range []func(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error{ daemon.reloadPlatform, daemon.reloadDebug, daemon.reloadMaxConcurrentDownloadsAndUploads, @@ -115,7 +117,7 @@ func (daemon *Daemon) Reload(conf *config.Config) error { *config.Config config.Proxies `json:"proxies"` }{ - Config: newCfg, + Config: &newCfg.Config, Proxies: config.Proxies{ HTTPProxy: config.MaskCredentials(newCfg.HTTPProxy), HTTPSProxy: config.MaskCredentials(newCfg.HTTPSProxy), @@ -141,7 +143,7 @@ func marshalAttributeSlice(v []string) string { // reloadDebug updates configuration with Debug option // and updates the passed attributes -func (daemon *Daemon) reloadDebug(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error { +func (daemon *Daemon) reloadDebug(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error { // update corresponding configuration if conf.IsValueSet("debug") { newCfg.Debug = conf.Debug @@ -153,7 +155,7 @@ func (daemon *Daemon) reloadDebug(txn *reloadTxn, newCfg, conf *config.Config, a // reloadMaxConcurrentDownloadsAndUploads updates configuration with max concurrent // download and upload options and updates the passed attributes -func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error { +func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error { // We always "reset" as the cost is lightweight and easy to maintain. newCfg.MaxConcurrentDownloads = config.DefaultMaxConcurrentDownloads newCfg.MaxConcurrentUploads = config.DefaultMaxConcurrentUploads @@ -184,7 +186,7 @@ func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(txn *reloadTxn, new // reloadMaxDownloadAttempts updates configuration with max concurrent // download attempts when a connection is lost and updates the passed attributes -func (daemon *Daemon) reloadMaxDownloadAttempts(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error { +func (daemon *Daemon) reloadMaxDownloadAttempts(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error { // We always "reset" as the cost is lightweight and easy to maintain. newCfg.MaxDownloadAttempts = config.DefaultDownloadAttempts if conf.IsValueSet("max-download-attempts") && conf.MaxDownloadAttempts != 0 { @@ -199,7 +201,7 @@ func (daemon *Daemon) reloadMaxDownloadAttempts(txn *reloadTxn, newCfg, conf *co // reloadShutdownTimeout updates configuration with daemon shutdown timeout option // and updates the passed attributes -func (daemon *Daemon) reloadShutdownTimeout(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error { +func (daemon *Daemon) reloadShutdownTimeout(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error { // update corresponding configuration if conf.IsValueSet("shutdown-timeout") { newCfg.ShutdownTimeout = conf.ShutdownTimeout @@ -213,7 +215,7 @@ func (daemon *Daemon) reloadShutdownTimeout(txn *reloadTxn, newCfg, conf *config // reloadLabels updates configuration with engine labels // and updates the passed attributes -func (daemon *Daemon) reloadLabels(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error { +func (daemon *Daemon) reloadLabels(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error { // update corresponding configuration if conf.IsValueSet("labels") { newCfg.Labels = conf.Labels @@ -226,7 +228,7 @@ func (daemon *Daemon) reloadLabels(txn *reloadTxn, newCfg, conf *config.Config, // reloadRegistryConfig updates the configuration with registry options // and updates the passed attributes. -func (daemon *Daemon) reloadRegistryConfig(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error { +func (daemon *Daemon) reloadRegistryConfig(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error { // Update corresponding configuration. if conf.IsValueSet("allow-nondistributable-artifacts") { newCfg.ServiceOptions.AllowNondistributableArtifacts = conf.AllowNondistributableArtifacts @@ -253,7 +255,7 @@ func (daemon *Daemon) reloadRegistryConfig(txn *reloadTxn, newCfg, conf *config. // reloadLiveRestore updates configuration with live restore option // and updates the passed attributes -func (daemon *Daemon) reloadLiveRestore(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error { +func (daemon *Daemon) reloadLiveRestore(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error { // update corresponding configuration if conf.IsValueSet("live-restore") { newCfg.LiveRestoreEnabled = conf.LiveRestoreEnabled @@ -265,7 +267,7 @@ func (daemon *Daemon) reloadLiveRestore(txn *reloadTxn, newCfg, conf *config.Con } // reloadNetworkDiagnosticPort updates the network controller starting the diagnostic if the config is valid -func (daemon *Daemon) reloadNetworkDiagnosticPort(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error { +func (daemon *Daemon) reloadNetworkDiagnosticPort(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error { txn.OnCommit(func() error { if conf == nil || daemon.netController == nil || !conf.IsValueSet("network-diagnostic-port") || conf.NetworkDiagnosticPort < 1 || conf.NetworkDiagnosticPort > 65535 { @@ -284,7 +286,7 @@ func (daemon *Daemon) reloadNetworkDiagnosticPort(txn *reloadTxn, newCfg, conf * } // reloadFeatures updates configuration with enabled/disabled features -func (daemon *Daemon) reloadFeatures(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error { +func (daemon *Daemon) reloadFeatures(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error { // update corresponding configuration // note that we allow features option to be entirely unset newCfg.Features = conf.Features diff --git a/daemon/reload_test.go b/daemon/reload_test.go index a959e2c47a..abd61c3ba6 100644 --- a/daemon/reload_test.go +++ b/daemon/reload_test.go @@ -27,7 +27,7 @@ func newDaemonForReloadT(t *testing.T, cfg *config.Config) *Daemon { var err error daemon.registryService, err = registry.NewService(registry.ServiceOptions{}) assert.Assert(t, err) - daemon.configStore.Store(cfg) + daemon.configStore.Store(&configStore{Config: *cfg}) return daemon } diff --git a/daemon/reload_unix.go b/daemon/reload_unix.go index 97c902be90..8b24bbd63c 100644 --- a/daemon/reload_unix.go +++ b/daemon/reload_unix.go @@ -11,15 +11,19 @@ import ( // reloadPlatform updates configuration with platform specific options // and updates the passed attributes -func (daemon *Daemon) reloadPlatform(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error { +func (daemon *Daemon) reloadPlatform(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error { if conf.DefaultRuntime != "" { newCfg.DefaultRuntime = conf.DefaultRuntime } if conf.IsValueSet("runtimes") { - newCfg.Runtimes = conf.Runtimes - txn.OnCommit(func() error { return daemon.initRuntimes(newCfg) }) + newCfg.Config.Runtimes = conf.Runtimes + } + configureRuntimes(&newCfg.Config) + var err error + newCfg.Runtimes, err = setupRuntimes(&newCfg.Config) + if err != nil { + return err } - configureRuntimes(newCfg) if conf.IsValueSet("default-shm-size") { newCfg.ShmSize = conf.ShmSize @@ -35,7 +39,7 @@ func (daemon *Daemon) reloadPlatform(txn *reloadTxn, newCfg, conf *config.Config // Update attributes var runtimeList bytes.Buffer - for name, rt := range newCfg.Runtimes { + for name, rt := range newCfg.Config.Runtimes { if runtimeList.Len() > 0 { runtimeList.WriteRune(' ') } diff --git a/daemon/reload_windows.go b/daemon/reload_windows.go index af218c8cf5..75cdb37dda 100644 --- a/daemon/reload_windows.go +++ b/daemon/reload_windows.go @@ -4,6 +4,6 @@ import "github.com/docker/docker/daemon/config" // reloadPlatform updates configuration with platform specific options // and updates the passed attributes -func (daemon *Daemon) reloadPlatform(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error { +func (daemon *Daemon) reloadPlatform(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error { return nil } diff --git a/daemon/restart.go b/daemon/restart.go index 1799c0fa16..522160ddb6 100644 --- a/daemon/restart.go +++ b/daemon/restart.go @@ -6,7 +6,6 @@ import ( containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" - "github.com/docker/docker/daemon/config" ) // ContainerRestart stops and starts a container. It attempts to @@ -31,7 +30,7 @@ func (daemon *Daemon) ContainerRestart(ctx context.Context, name string, options // container. When stopping, wait for the given duration in seconds to // gracefully stop, before forcefully terminating the container. If // given a negative duration, wait forever for a graceful stop. -func (daemon *Daemon) containerRestart(ctx context.Context, daemonCfg *config.Config, container *container.Container, options containertypes.StopOptions) error { +func (daemon *Daemon) containerRestart(ctx context.Context, daemonCfg *configStore, container *container.Container, options containertypes.StopOptions) error { // Determine isolation. If not specified in the hostconfig, use daemon default. actualIsolation := container.HostConfig.Isolation if containertypes.Isolation.IsDefault(actualIsolation) { diff --git a/daemon/runtime_unix.go b/daemon/runtime_unix.go index 20c804b394..402f6eebb4 100644 --- a/daemon/runtime_unix.go +++ b/daemon/runtime_unix.go @@ -4,18 +4,24 @@ package daemon import ( "bytes" + "crypto/sha256" + "encoding/base32" "encoding/json" "fmt" + "io" "os" "os/exec" "path/filepath" "strings" + "github.com/containerd/containerd/plugin" v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options" "github.com/docker/docker/api/types" "github.com/docker/docker/daemon/config" "github.com/docker/docker/errdefs" "github.com/docker/docker/libcontainerd/shimopts" + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/system" "github.com/opencontainers/runtime-spec/specs-go/features" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -23,10 +29,21 @@ import ( const ( defaultRuntimeName = "runc" - - linuxShimV2 = "io.containerd.runc.v2" ) +type shimConfig struct { + Shim string + Opts interface{} + Features *features.Features + + // Check if the ShimConfig is valid given the current state of the system. + PreflightCheck func() error +} + +type runtimes struct { + configured map[string]*shimConfig +} + func configureRuntimes(conf *config.Config) { if conf.DefaultRuntime == "" { conf.DefaultRuntime = config.StockRuntimeName @@ -34,13 +51,13 @@ func configureRuntimes(conf *config.Config) { if conf.Runtimes == nil { conf.Runtimes = make(map[string]types.Runtime) } - conf.Runtimes[config.LinuxV2RuntimeName] = types.Runtime{Path: defaultRuntimeName, ShimConfig: defaultV2ShimConfig(conf, defaultRuntimeName)} + conf.Runtimes[config.LinuxV2RuntimeName] = types.Runtime{Path: defaultRuntimeName} conf.Runtimes[config.StockRuntimeName] = conf.Runtimes[config.LinuxV2RuntimeName] } -func defaultV2ShimConfig(conf *config.Config, runtimePath string) *types.ShimConfig { - return &types.ShimConfig{ - Binary: linuxShimV2, +func defaultV2ShimConfig(conf *config.Config, runtimePath string) *shimConfig { + shim := &shimConfig{ + Shim: plugin.RuntimeRuncV2, Opts: &v2runcoptions.Options{ BinaryName: runtimePath, Root: filepath.Join(conf.ExecRoot, "runtime-"+defaultRuntimeName), @@ -48,138 +65,142 @@ func defaultV2ShimConfig(conf *config.Config, runtimePath string) *types.ShimCon NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "", }, } -} -func (daemon *Daemon) loadRuntimes() error { - return daemon.initRuntimes(daemon.config()) -} - -func (daemon *Daemon) initRuntimes(cfg *config.Config) (err error) { - runtimeDir := filepath.Join(cfg.Root, "runtimes") - runtimeOldDir := runtimeDir + "-old" - // Remove old temp directory if any - os.RemoveAll(runtimeOldDir) - tmpDir, err := os.MkdirTemp(cfg.Root, "gen-runtimes") - if err != nil { - return errors.Wrap(err, "failed to get temp dir to generate runtime scripts") + var featuresStderr bytes.Buffer + featuresCmd := exec.Command(runtimePath, "features") + featuresCmd.Stderr = &featuresStderr + if featuresB, err := featuresCmd.Output(); err != nil { + logrus.WithError(err).Warnf("Failed to run %v: %q", featuresCmd.Args, featuresStderr.String()) + } else { + var features features.Features + if jsonErr := json.Unmarshal(featuresB, &features); jsonErr != nil { + logrus.WithError(err).Warnf("Failed to unmarshal the output of %v as a JSON", featuresCmd.Args) + } else { + shim.Features = &features + } } - defer func() { - if err != nil { - if err1 := os.RemoveAll(tmpDir); err1 != nil { - logrus.WithError(err1).WithField("dir", tmpDir). - Warn("failed to remove tmp dir") - } - return - } - if err = os.Rename(runtimeDir, runtimeOldDir); err != nil { - logrus.WithError(err).WithField("dir", runtimeDir). - Warn("failed to rename runtimes dir to old. Will try to removing it") - if err = os.RemoveAll(runtimeDir); err != nil { - logrus.WithError(err).WithField("dir", runtimeDir). - Warn("failed to remove old runtimes dir") - return - } - } - if err = os.Rename(tmpDir, runtimeDir); err != nil { - err = errors.Wrap(err, "failed to setup runtimes dir, new containers may not start") - return - } - if err = os.RemoveAll(runtimeOldDir); err != nil { - logrus.WithError(err).WithField("dir", runtimeOldDir). - Warn("failed to remove old runtimes dir") - } - }() + return shim +} - for name := range cfg.Runtimes { - rt := cfg.Runtimes[name] +func runtimeScriptsDir(cfg *config.Config) string { + return filepath.Join(cfg.Root, "runtimes") +} + +// initRuntimesDir creates a fresh directory where we'll store the runtime +// scripts (i.e. in order to support runtimeArgs). +func initRuntimesDir(cfg *config.Config) error { + runtimeDir := runtimeScriptsDir(cfg) + if err := os.RemoveAll(runtimeDir); err != nil { + return err + } + return system.MkdirAll(runtimeDir, 0700) +} + +func setupRuntimes(cfg *config.Config) (runtimes, error) { + newrt := runtimes{ + configured: make(map[string]*shimConfig), + } + + dir := runtimeScriptsDir(cfg) + for name, rt := range cfg.Runtimes { + var c *shimConfig if rt.Path == "" && rt.Type == "" { - return errors.Errorf("runtime %s: either a runtimeType or a path must be configured", name) + return runtimes{}, errors.Errorf("runtime %s: either a runtimeType or a path must be configured", name) } if rt.Path != "" { if rt.Type != "" { - return errors.Errorf("runtime %s: cannot configure both path and runtimeType for the same runtime", name) + return runtimes{}, errors.Errorf("runtime %s: cannot configure both path and runtimeType for the same runtime", name) } if len(rt.Options) > 0 { - return errors.Errorf("runtime %s: options cannot be used with a path runtime", name) + return runtimes{}, errors.Errorf("runtime %s: options cannot be used with a path runtime", name) } - if len(rt.Args) > 0 { - script := filepath.Join(tmpDir, name) - content := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", rt.Path, strings.Join(rt.Args, " ")) - if err := os.WriteFile(script, []byte(content), 0700); err != nil { - return err + binaryName := rt.Path + needsWrapper := len(rt.Args) > 0 + if needsWrapper { + var err error + binaryName, err = wrapRuntime(dir, name, rt.Path, rt.Args) + if err != nil { + return runtimes{}, err } } - rt.ShimConfig = defaultV2ShimConfig(cfg, daemon.rewriteRuntimePath(cfg, name, rt.Path, rt.Args)) - var featuresStderr bytes.Buffer - featuresCmd := exec.Command(rt.Path, append(rt.Args, "features")...) - featuresCmd.Stderr = &featuresStderr - if featuresB, err := featuresCmd.Output(); err != nil { - logrus.WithError(err).Warnf("Failed to run %v: %q", featuresCmd.Args, featuresStderr.String()) - } else { - var features features.Features - if jsonErr := json.Unmarshal(featuresB, &features); jsonErr != nil { - logrus.WithError(err).Warnf("Failed to unmarshal the output of %v as a JSON", featuresCmd.Args) - } else { - rt.Features = &features + c = defaultV2ShimConfig(cfg, binaryName) + if needsWrapper { + path := rt.Path + c.PreflightCheck = func() error { + // Check that the runtime path actually exists so that we can return a well known error. + _, err := exec.LookPath(path) + return errors.Wrap(err, "error while looking up the specified runtime path") } } } else { if len(rt.Args) > 0 { - return errors.Errorf("runtime %s: args cannot be used with a runtimeType runtime", name) + return runtimes{}, errors.Errorf("runtime %s: args cannot be used with a runtimeType runtime", name) } // Unlike implicit runtimes, there is no restriction on configuring a shim by path. - rt.ShimConfig = &types.ShimConfig{Binary: rt.Type} + c = &shimConfig{Shim: rt.Type} if len(rt.Options) > 0 { // It has to be a pointer type or there'll be a panic in containerd/typeurl when we try to start the container. - rt.ShimConfig.Opts, err = shimopts.Generate(rt.Type, rt.Options) + var err error + c.Opts, err = shimopts.Generate(rt.Type, rt.Options) if err != nil { - return errors.Wrapf(err, "runtime %v", name) + return runtimes{}, errors.Wrapf(err, "runtime %v", name) } } } - cfg.Runtimes[name] = rt + newrt.configured[name] = c + } + + return newrt, nil +} + +// A non-standard Base32 encoding which lacks vowels to avoid accidentally +// spelling naughty words. Don't use this to encode any data which requires +// compatibility with anything outside of the currently-running process. +var base32Disemvoweled = base32.NewEncoding("0123456789BCDFGHJKLMNPQRSTVWXYZ-") + +// wrapRuntime writes a shell script to dir which will execute binary with args +// concatenated to the script's argv. This is needed because the +// io.containerd.runc.v2 shim has no options for passing extra arguments to the +// runtime binary. +func wrapRuntime(dir, name, binary string, args []string) (string, error) { + var wrapper bytes.Buffer + sum := sha256.New() + _, _ = fmt.Fprintf(io.MultiWriter(&wrapper, sum), "#!/bin/sh\n%s %s $@\n", binary, strings.Join(args, " ")) + // Generate a consistent name for the wrapper script derived from the + // contents so that multiple wrapper scripts can coexist with the same + // base name. The existing scripts might still be referenced by running + // containers. + suffix := base32Disemvoweled.EncodeToString(sum.Sum(nil)) + scriptPath := filepath.Join(dir, name+"."+suffix) + if err := ioutils.AtomicWriteFile(scriptPath, wrapper.Bytes(), 0700); err != nil { + return "", err + } + return scriptPath, nil +} + +func (r *runtimes) Get(name string) (string, interface{}, error) { + rt := r.configured[name] + if rt != nil { + if rt.PreflightCheck != nil { + if err := rt.PreflightCheck(); err != nil { + return "", nil, err + } + } + return rt.Shim, rt.Opts, nil + } + + if !config.IsPermissibleC8dRuntimeName(name) { + return "", nil, errdefs.InvalidParameter(errors.Errorf("unknown or invalid runtime name: %s", name)) + } + return name, nil, nil +} + +func (r *runtimes) Features(name string) *features.Features { + rt := r.configured[name] + if rt != nil { + return rt.Features } return nil } - -// rewriteRuntimePath is used for runtimes which have custom arguments supplied. -// This is needed because the containerd API only calls the OCI runtime binary, there is no options for extra arguments. -// To support this case, the daemon wraps the specified runtime in a script that passes through those arguments. -func (daemon *Daemon) rewriteRuntimePath(cfg *config.Config, name, p string, args []string) string { - if len(args) == 0 { - return p - } - - return filepath.Join(cfg.Root, "runtimes", name) -} - -func (daemon *Daemon) getRuntime(cfg *config.Config, name string) (shim string, opts interface{}, err error) { - rt := cfg.GetRuntime(name) - if rt == nil { - if !config.IsPermissibleC8dRuntimeName(name) { - return "", nil, errdefs.InvalidParameter(errors.Errorf("unknown or invalid runtime name: %s", name)) - } - return name, nil, nil - } - - if len(rt.Args) > 0 { - // Check that the path of the runtime which the script wraps actually exists so - // that we can return a well known error which references the configured path - // instead of the wrapper script's. - if _, err := exec.LookPath(rt.Path); err != nil { - return "", nil, errors.Wrap(err, "error while looking up the specified runtime path") - } - } - - if rt.ShimConfig == nil { - // Should never happen as daemon.initRuntimes always sets - // ShimConfig and config reloading is synchronized. - err := errdefs.System(errors.Errorf("BUG: runtime %s: rt.ShimConfig == nil", name)) - logrus.Error(err) - return "", nil, err - } - - return rt.ShimConfig.Binary, rt.ShimConfig.Opts, nil -} diff --git a/daemon/runtime_unix_test.go b/daemon/runtime_unix_test.go index dae8a213b9..582c77d7a5 100644 --- a/daemon/runtime_unix_test.go +++ b/daemon/runtime_unix_test.go @@ -3,8 +3,9 @@ package daemon import ( + "io/fs" "os" - "path/filepath" + "strings" "testing" "github.com/containerd/containerd/plugin" @@ -88,11 +89,9 @@ func TestInitRuntimes_InvalidConfigs(t *testing.T) { assert.NilError(t, err) cfg.Root = t.TempDir() cfg.Runtimes["myruntime"] = tt.runtime - d := &Daemon{} - d.configStore.Store(cfg) - assert.Assert(t, os.Mkdir(filepath.Join(d.config().Root, "runtimes"), 0700)) + assert.Assert(t, initRuntimesDir(cfg)) - err = d.initRuntimes(d.config()) + _, err = setupRuntimes(cfg) assert.Check(t, is.ErrorContains(err, tt.expectErr)) }) } @@ -127,7 +126,6 @@ func TestGetRuntime(t *testing.T) { assert.NilError(t, err) cfg.Root = t.TempDir() - assert.Assert(t, os.Mkdir(filepath.Join(cfg.Root, "runtimes"), 0700)) cfg.Runtimes = map[string]types.Runtime{ configuredRtName: configuredRuntime, rtWithArgsName: rtWithArgs, @@ -136,41 +134,43 @@ func TestGetRuntime(t *testing.T) { configuredShimByPathName: configuredShimByPath, } configureRuntimes(cfg) + assert.NilError(t, initRuntimesDir(cfg)) + runtimes, err := setupRuntimes(cfg) + assert.NilError(t, err) - d := &Daemon{} - d.configStore.Store(cfg) - assert.Assert(t, d.loadRuntimes()) - - stockRuntime, ok := cfg.Runtimes[config.StockRuntimeName] + stockRuntime, ok := runtimes.configured[config.StockRuntimeName] assert.Assert(t, ok, "stock runtime could not be found (test needs to be updated)") + stockRuntime.Features = nil - configdOpts := *stockRuntime.ShimConfig.Opts.(*v2runcoptions.Options) + configdOpts := *stockRuntime.Opts.(*v2runcoptions.Options) configdOpts.BinaryName = configuredRuntime.Path + wantConfigdRuntime := &shimConfig{ + Shim: stockRuntime.Shim, + Opts: &configdOpts, + } for _, tt := range []struct { name, runtime string - wantShim string - wantOpts interface{} + want *shimConfig }{ { - name: "StockRuntime", - runtime: config.StockRuntimeName, - wantShim: stockRuntime.ShimConfig.Binary, - wantOpts: stockRuntime.ShimConfig.Opts, + name: "StockRuntime", + runtime: config.StockRuntimeName, + want: stockRuntime, }, { - name: "ShimName", - runtime: "io.containerd.my-shim.v42", - wantShim: "io.containerd.my-shim.v42", + name: "ShimName", + runtime: "io.containerd.my-shim.v42", + want: &shimConfig{Shim: "io.containerd.my-shim.v42"}, }, { // containerd is pretty loose about the format of runtime names. Perhaps too // loose. The only requirements are that the name contain a dot and (depending // on the containerd version) not start with a dot. It does not enforce any // particular format of the dot-delimited components of the name. - name: "VersionlessShimName", - runtime: "io.containerd.my-shim", - wantShim: "io.containerd.my-shim", + name: "VersionlessShimName", + runtime: "io.containerd.my-shim", + want: &shimConfig{Shim: "io.containerd.my-shim"}, }, { name: "IllformedShimName", @@ -193,50 +193,152 @@ func TestGetRuntime(t *testing.T) { runtime: "my/io.containerd.runc.v2", }, { - name: "ConfiguredRuntime", - runtime: configuredRtName, - wantShim: stockRuntime.ShimConfig.Binary, - wantOpts: &configdOpts, + name: "ConfiguredRuntime", + runtime: configuredRtName, + want: wantConfigdRuntime, }, { - name: "RuntimeWithArgs", - runtime: rtWithArgsName, - wantShim: stockRuntime.ShimConfig.Binary, - wantOpts: defaultV2ShimConfig( - d.config(), - d.rewriteRuntimePath( - d.config(), - rtWithArgsName, - rtWithArgs.Path, - rtWithArgs.Args)).Opts, + name: "ShimWithOpts", + runtime: shimWithOptsName, + want: &shimConfig{ + Shim: shimWithOpts.Type, + Opts: &v2runcoptions.Options{IoUid: 42}, + }, }, { - name: "ShimWithOpts", - runtime: shimWithOptsName, - wantShim: shimWithOpts.Type, - wantOpts: &v2runcoptions.Options{IoUid: 42}, + name: "ShimAlias", + runtime: shimAliasName, + want: &shimConfig{Shim: shimAlias.Type}, }, { - name: "ShimAlias", - runtime: shimAliasName, - wantShim: shimAlias.Type, - }, - { - name: "ConfiguredShimByPath", - runtime: configuredShimByPathName, - wantShim: configuredShimByPath.Type, + name: "ConfiguredShimByPath", + runtime: configuredShimByPathName, + want: &shimConfig{Shim: configuredShimByPath.Type}, }, } { tt := tt t.Run(tt.name, func(t *testing.T) { - gotShim, gotOpts, err := d.getRuntime(cfg, tt.runtime) - assert.Check(t, is.Equal(gotShim, tt.wantShim)) - assert.Check(t, is.DeepEqual(gotOpts, tt.wantOpts)) - if tt.wantShim != "" { + shim, opts, err := runtimes.Get(tt.runtime) + if tt.want != nil { assert.Check(t, err) + got := &shimConfig{Shim: shim, Opts: opts} + assert.Check(t, is.DeepEqual(got, tt.want)) } else { - assert.Check(t, errdefs.IsInvalidParameter(err)) + assert.Check(t, is.Equal(shim, "")) + assert.Check(t, is.Nil(opts)) + assert.Check(t, errdefs.IsInvalidParameter(err), "[%T] %[1]v", err) } }) } + t.Run("RuntimeWithArgs", func(t *testing.T) { + shim, opts, err := runtimes.Get(rtWithArgsName) + assert.Check(t, err) + assert.Check(t, is.Equal(shim, stockRuntime.Shim)) + runcopts, ok := opts.(*v2runcoptions.Options) + if assert.Check(t, ok, "runtimes.Get() opts = type %T, want *v2runcoptions.Options", opts) { + wrapper, err := os.ReadFile(runcopts.BinaryName) + if assert.Check(t, err) { + assert.Check(t, is.Contains(string(wrapper), + strings.Join(append([]string{rtWithArgs.Path}, rtWithArgs.Args...), " "))) + } + } + }) +} + +func TestGetRuntime_PreflightCheck(t *testing.T) { + cfg, err := config.New() + assert.NilError(t, err) + + cfg.Root = t.TempDir() + cfg.Runtimes = map[string]types.Runtime{ + "path-only": { + Path: "/usr/local/bin/file-not-found", + }, + "with-args": { + Path: "/usr/local/bin/file-not-found", + Args: []string{"--arg"}, + }, + } + assert.NilError(t, initRuntimesDir(cfg)) + runtimes, err := setupRuntimes(cfg) + assert.NilError(t, err, "runtime paths should not be validated during setupRuntimes()") + + t.Run("PathOnly", func(t *testing.T) { + _, _, err := runtimes.Get("path-only") + assert.NilError(t, err, "custom runtimes without wrapper scripts should not have pre-flight checks") + }) + t.Run("WithArgs", func(t *testing.T) { + _, _, err := runtimes.Get("with-args") + assert.ErrorIs(t, err, fs.ErrNotExist) + }) +} + +// TestRuntimeWrapping checks that reloading runtime config does not delete or +// modify existing wrapper scripts, which could break lifecycle management of +// existing containers. +func TestRuntimeWrapping(t *testing.T) { + cfg, err := config.New() + assert.NilError(t, err) + cfg.Root = t.TempDir() + cfg.Runtimes = map[string]types.Runtime{ + "change-args": { + Path: "/bin/true", + Args: []string{"foo", "bar"}, + }, + "dupe": { + Path: "/bin/true", + Args: []string{"foo", "bar"}, + }, + "change-path": { + Path: "/bin/true", + Args: []string{"baz"}, + }, + "drop-args": { + Path: "/bin/true", + Args: []string{"some", "arguments"}, + }, + "goes-away": { + Path: "/bin/true", + Args: []string{"bye"}, + }, + } + assert.NilError(t, initRuntimesDir(cfg)) + rt, err := setupRuntimes(cfg) + assert.Check(t, err) + + type WrapperInfo struct{ BinaryName, Content string } + wrappers := make(map[string]WrapperInfo) + for name := range cfg.Runtimes { + _, opts, err := rt.Get(name) + if assert.Check(t, err, "rt.Get(%q)", name) { + binary := opts.(*v2runcoptions.Options).BinaryName + content, err := os.ReadFile(binary) + assert.Check(t, err, "could not read wrapper script contents for runtime %q", binary) + wrappers[name] = WrapperInfo{BinaryName: binary, Content: string(content)} + } + } + + cfg.Runtimes["change-args"] = types.Runtime{ + Path: cfg.Runtimes["change-args"].Path, + Args: []string{"baz", "quux"}, + } + cfg.Runtimes["change-path"] = types.Runtime{ + Path: "/bin/false", + Args: cfg.Runtimes["change-path"].Args, + } + cfg.Runtimes["drop-args"] = types.Runtime{ + Path: cfg.Runtimes["drop-args"].Path, + } + delete(cfg.Runtimes, "goes-away") + + _, err = setupRuntimes(cfg) + assert.Check(t, err) + + for name, info := range wrappers { + t.Run(name, func(t *testing.T) { + content, err := os.ReadFile(info.BinaryName) + assert.NilError(t, err) + assert.DeepEqual(t, info.Content, string(content)) + }) + } } diff --git a/daemon/runtime_windows.go b/daemon/runtime_windows.go index cbbe0b577b..7c79cb5a25 100644 --- a/daemon/runtime_windows.go +++ b/daemon/runtime_windows.go @@ -6,6 +6,16 @@ import ( "github.com/docker/docker/daemon/config" ) -func (daemon *Daemon) getRuntime(cfg *config.Config, name string) (shim string, opts interface{}, err error) { +type runtimes struct{} + +func (r *runtimes) Get(name string) (string, interface{}, error) { return "", nil, errors.New("not implemented") } + +func initRuntimesDir(*config.Config) error { + return nil +} + +func setupRuntimes(*config.Config) (runtimes, error) { + return runtimes{}, nil +} diff --git a/daemon/start.go b/daemon/start.go index d300cedaf9..ac14a906ce 100644 --- a/daemon/start.go +++ b/daemon/start.go @@ -9,7 +9,6 @@ import ( "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" - "github.com/docker/docker/daemon/config" "github.com/docker/docker/errdefs" "github.com/docker/docker/libcontainerd" "github.com/pkg/errors" @@ -57,7 +56,7 @@ func (daemon *Daemon) ContainerStart(ctx context.Context, name string, hostConfi if hostConfig != nil { logrus.Warn("DEPRECATED: Setting host configuration options when the container starts is deprecated and has been removed in Docker 1.12") oldNetworkMode := ctr.HostConfig.NetworkMode - if err := daemon.setSecurityOptions(daemonCfg, ctr, hostConfig); err != nil { + if err := daemon.setSecurityOptions(&daemonCfg.Config, ctr, hostConfig); err != nil { return errdefs.InvalidParameter(err) } if err := daemon.mergeAndVerifyLogConfig(&hostConfig.LogConfig); err != nil { @@ -91,7 +90,7 @@ func (daemon *Daemon) ContainerStart(ctx context.Context, name string, hostConfi // Adapt for old containers in case we have updates in this function and // old containers never have chance to call the new function in create stage. if hostConfig != nil { - if err := daemon.adaptContainerSettings(daemonCfg, ctr.HostConfig, false); err != nil { + if err := daemon.adaptContainerSettings(&daemonCfg.Config, ctr.HostConfig, false); err != nil { return errdefs.InvalidParameter(err) } } @@ -102,7 +101,7 @@ func (daemon *Daemon) ContainerStart(ctx context.Context, name string, hostConfi // container needs, such as storage and networking, as well as links // between containers. The container is left waiting for a signal to // begin running. -func (daemon *Daemon) containerStart(ctx context.Context, daemonCfg *config.Config, container *container.Container, checkpoint string, checkpointDir string, resetRestartManager bool) (retErr error) { +func (daemon *Daemon) containerStart(ctx context.Context, daemonCfg *configStore, container *container.Container, checkpoint string, checkpointDir string, resetRestartManager bool) (retErr error) { start := time.Now() container.Lock() defer container.Unlock() @@ -138,7 +137,7 @@ func (daemon *Daemon) containerStart(ctx context.Context, daemonCfg *config.Conf // if containers AutoRemove flag is set, remove it after clean up if container.HostConfig.AutoRemove { container.Unlock() - if err := daemon.containerRm(daemonCfg, container.ID, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil { + if err := daemon.containerRm(&daemonCfg.Config, container.ID, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil { logrus.Errorf("can't remove container %s: %v", container.ID, err) } container.Lock() @@ -150,7 +149,7 @@ func (daemon *Daemon) containerStart(ctx context.Context, daemonCfg *config.Conf return err } - if err := daemon.initializeNetworking(daemonCfg, container); err != nil { + if err := daemon.initializeNetworking(&daemonCfg.Config, container); err != nil { return err } diff --git a/daemon/start_unix.go b/daemon/start_unix.go index 24999cb880..8d54d2a353 100644 --- a/daemon/start_unix.go +++ b/daemon/start_unix.go @@ -4,21 +4,20 @@ package daemon // import "github.com/docker/docker/daemon" import ( "github.com/docker/docker/container" - "github.com/docker/docker/daemon/config" ) // getLibcontainerdCreateOptions callers must hold a lock on the container -func (daemon *Daemon) getLibcontainerdCreateOptions(daemonCfg *config.Config, container *container.Container) (string, interface{}, error) { +func (daemon *Daemon) getLibcontainerdCreateOptions(daemonCfg *configStore, container *container.Container) (string, interface{}, error) { // Ensure a runtime has been assigned to this container if container.HostConfig.Runtime == "" { container.HostConfig.Runtime = daemonCfg.DefaultRuntime container.CheckpointTo(daemon.containersReplica) } - binary, opts, err := daemon.getRuntime(daemonCfg, container.HostConfig.Runtime) + shim, opts, err := daemonCfg.Runtimes.Get(container.HostConfig.Runtime) if err != nil { return "", nil, setExitCodeFromError(container.SetExitCode, err) } - return binary, opts, nil + return shim, opts, nil } diff --git a/daemon/start_windows.go b/daemon/start_windows.go index 4238dcb338..3c87f004c8 100644 --- a/daemon/start_windows.go +++ b/daemon/start_windows.go @@ -3,11 +3,10 @@ package daemon // import "github.com/docker/docker/daemon" import ( "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options" "github.com/docker/docker/container" - "github.com/docker/docker/daemon/config" "github.com/docker/docker/pkg/system" ) -func (daemon *Daemon) getLibcontainerdCreateOptions(*config.Config, *container.Container) (string, interface{}, error) { +func (daemon *Daemon) getLibcontainerdCreateOptions(*configStore, *container.Container) (string, interface{}, error) { if system.ContainerdRuntimeSupported() { opts := &options.Options{} return "io.containerd.runhcs.v1", opts, nil