daemon.go 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524
  1. // Package daemon exposes the functions that occur on the host server
  2. // that the Docker daemon is running.
  3. //
  4. // In implementing the various functions of the daemon, there is often
  5. // a method-specific struct for configuring the runtime behavior.
  6. package daemon // import "github.com/docker/docker/daemon"
  7. import (
  8. "context"
  9. "fmt"
  10. "net"
  11. "net/url"
  12. "os"
  13. "path"
  14. "path/filepath"
  15. "runtime"
  16. "strings"
  17. "sync"
  18. "time"
  19. "github.com/containerd/containerd"
  20. "github.com/containerd/containerd/defaults"
  21. "github.com/containerd/containerd/pkg/dialer"
  22. "github.com/containerd/containerd/pkg/userns"
  23. "github.com/containerd/containerd/remotes/docker"
  24. dist "github.com/docker/distribution"
  25. "github.com/docker/distribution/reference"
  26. "github.com/docker/docker/api/types"
  27. containertypes "github.com/docker/docker/api/types/container"
  28. registrytypes "github.com/docker/docker/api/types/registry"
  29. "github.com/docker/docker/api/types/swarm"
  30. "github.com/docker/docker/api/types/volume"
  31. "github.com/docker/docker/builder"
  32. "github.com/docker/docker/container"
  33. executorpkg "github.com/docker/docker/daemon/cluster/executor"
  34. "github.com/docker/docker/daemon/config"
  35. ctrd "github.com/docker/docker/daemon/containerd"
  36. "github.com/docker/docker/daemon/events"
  37. _ "github.com/docker/docker/daemon/graphdriver/register" // register graph drivers
  38. "github.com/docker/docker/daemon/images"
  39. dlogger "github.com/docker/docker/daemon/logger"
  40. "github.com/docker/docker/daemon/network"
  41. "github.com/docker/docker/daemon/stats"
  42. "github.com/docker/docker/distribution"
  43. dmetadata "github.com/docker/docker/distribution/metadata"
  44. "github.com/docker/docker/dockerversion"
  45. "github.com/docker/docker/errdefs"
  46. "github.com/docker/docker/image"
  47. "github.com/docker/docker/layer"
  48. libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
  49. "github.com/docker/docker/libnetwork"
  50. "github.com/docker/docker/libnetwork/cluster"
  51. nwconfig "github.com/docker/docker/libnetwork/config"
  52. "github.com/docker/docker/pkg/authorization"
  53. "github.com/docker/docker/pkg/fileutils"
  54. "github.com/docker/docker/pkg/idtools"
  55. "github.com/docker/docker/pkg/plugingetter"
  56. "github.com/docker/docker/pkg/sysinfo"
  57. "github.com/docker/docker/pkg/system"
  58. "github.com/docker/docker/plugin"
  59. pluginexec "github.com/docker/docker/plugin/executor/containerd"
  60. refstore "github.com/docker/docker/reference"
  61. "github.com/docker/docker/registry"
  62. "github.com/docker/docker/runconfig"
  63. volumesservice "github.com/docker/docker/volume/service"
  64. "github.com/moby/buildkit/util/resolver"
  65. resolverconfig "github.com/moby/buildkit/util/resolver/config"
  66. "github.com/moby/locker"
  67. "github.com/pkg/errors"
  68. "github.com/sirupsen/logrus"
  69. "go.etcd.io/bbolt"
  70. "golang.org/x/sync/semaphore"
  71. "google.golang.org/grpc"
  72. "google.golang.org/grpc/backoff"
  73. "google.golang.org/grpc/credentials/insecure"
  74. "resenje.org/singleflight"
  75. )
  76. // Daemon holds information about the Docker daemon.
  77. type Daemon struct {
  78. id string
  79. repository string
  80. containers container.Store
  81. containersReplica *container.ViewDB
  82. execCommands *container.ExecStore
  83. imageService ImageService
  84. configStore *config.Config
  85. statsCollector *stats.Collector
  86. defaultLogConfig containertypes.LogConfig
  87. registryService *registry.Service
  88. EventsService *events.Events
  89. netController *libnetwork.Controller
  90. volumes *volumesservice.VolumesService
  91. root string
  92. sysInfoOnce sync.Once
  93. sysInfo *sysinfo.SysInfo
  94. shutdown bool
  95. idMapping idtools.IdentityMapping
  96. PluginStore *plugin.Store // TODO: remove
  97. pluginManager *plugin.Manager
  98. linkIndex *linkIndex
  99. containerdCli *containerd.Client
  100. containerd libcontainerdtypes.Client
  101. defaultIsolation containertypes.Isolation // Default isolation mode on Windows
  102. clusterProvider cluster.Provider
  103. cluster Cluster
  104. genericResources []swarm.GenericResource
  105. metricsPluginListener net.Listener
  106. ReferenceStore refstore.Store
  107. machineMemory uint64
  108. seccompProfile []byte
  109. seccompProfilePath string
  110. usageContainers singleflight.Group[struct{}, []*types.Container]
  111. usageImages singleflight.Group[struct{}, []*types.ImageSummary]
  112. usageVolumes singleflight.Group[struct{}, []*volume.Volume]
  113. usageLayer singleflight.Group[struct{}, int64]
  114. pruneRunning int32
  115. hosts map[string]bool // hosts stores the addresses the daemon is listening on
  116. startupDone chan struct{}
  117. attachmentStore network.AttachmentStore
  118. attachableNetworkLock *locker.Locker
  119. // This is used for Windows which doesn't currently support running on containerd
  120. // It stores metadata for the content store (used for manifest caching)
  121. // This needs to be closed on daemon exit
  122. mdDB *bbolt.DB
  123. }
  124. // StoreHosts stores the addresses the daemon is listening on
  125. func (daemon *Daemon) StoreHosts(hosts []string) {
  126. if daemon.hosts == nil {
  127. daemon.hosts = make(map[string]bool)
  128. }
  129. for _, h := range hosts {
  130. daemon.hosts[h] = true
  131. }
  132. }
  133. // HasExperimental returns whether the experimental features of the daemon are enabled or not
  134. func (daemon *Daemon) HasExperimental() bool {
  135. return daemon.configStore != nil && daemon.configStore.Experimental
  136. }
  137. // Features returns the features map from configStore
  138. func (daemon *Daemon) Features() *map[string]bool {
  139. return &daemon.configStore.Features
  140. }
  141. // UsesSnapshotter returns true if feature flag to use containerd snapshotter is enabled
  142. func (daemon *Daemon) UsesSnapshotter() bool {
  143. // TEST_INTEGRATION_USE_SNAPSHOTTER is used for integration tests only.
  144. if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" {
  145. return true
  146. }
  147. if daemon.configStore.Features != nil {
  148. if b, ok := daemon.configStore.Features["containerd-snapshotter"]; ok {
  149. return b
  150. }
  151. }
  152. return false
  153. }
  154. // RegistryHosts returns registry configuration in containerd resolvers format
  155. func (daemon *Daemon) RegistryHosts() docker.RegistryHosts {
  156. var (
  157. registryKey = "docker.io"
  158. mirrors = make([]string, len(daemon.configStore.Mirrors))
  159. m = map[string]resolverconfig.RegistryConfig{}
  160. )
  161. // must trim "https://" or "http://" prefix
  162. for i, v := range daemon.configStore.Mirrors {
  163. if uri, err := url.Parse(v); err == nil {
  164. v = uri.Host
  165. }
  166. mirrors[i] = v
  167. }
  168. // set mirrors for default registry
  169. m[registryKey] = resolverconfig.RegistryConfig{Mirrors: mirrors}
  170. for _, v := range daemon.configStore.InsecureRegistries {
  171. u, err := url.Parse(v)
  172. if err != nil && !strings.HasPrefix(v, "http://") && !strings.HasPrefix(v, "https://") {
  173. originalErr := err
  174. u, err = url.Parse("http://" + v)
  175. if err != nil {
  176. err = originalErr
  177. }
  178. }
  179. c := resolverconfig.RegistryConfig{}
  180. if err == nil {
  181. v = u.Host
  182. t := true
  183. if u.Scheme == "http" {
  184. c.PlainHTTP = &t
  185. } else {
  186. c.Insecure = &t
  187. }
  188. }
  189. m[v] = c
  190. }
  191. for k, v := range m {
  192. v.TLSConfigDir = []string{registry.HostCertsDir(k)}
  193. m[k] = v
  194. }
  195. certsDir := registry.CertsDir()
  196. if fis, err := os.ReadDir(certsDir); err == nil {
  197. for _, fi := range fis {
  198. if _, ok := m[fi.Name()]; !ok {
  199. m[fi.Name()] = resolverconfig.RegistryConfig{
  200. TLSConfigDir: []string{filepath.Join(certsDir, fi.Name())},
  201. }
  202. }
  203. }
  204. }
  205. return resolver.NewRegistryConfig(m)
  206. }
  207. // layerAccessor may be implemented by ImageService
  208. type layerAccessor interface {
  209. GetLayerByID(cid string) (layer.RWLayer, error)
  210. }
  211. func (daemon *Daemon) restore() error {
  212. var mapLock sync.Mutex
  213. containers := make(map[string]*container.Container)
  214. logrus.Info("Loading containers: start.")
  215. dir, err := os.ReadDir(daemon.repository)
  216. if err != nil {
  217. return err
  218. }
  219. // parallelLimit is the maximum number of parallel startup jobs that we
  220. // allow (this is the limited used for all startup semaphores). The multipler
  221. // (128) was chosen after some fairly significant benchmarking -- don't change
  222. // it unless you've tested it significantly (this value is adjusted if
  223. // RLIMIT_NOFILE is small to avoid EMFILE).
  224. parallelLimit := adjustParallelLimit(len(dir), 128*runtime.NumCPU())
  225. // Re-used for all parallel startup jobs.
  226. var group sync.WaitGroup
  227. sem := semaphore.NewWeighted(int64(parallelLimit))
  228. for _, v := range dir {
  229. group.Add(1)
  230. go func(id string) {
  231. defer group.Done()
  232. _ = sem.Acquire(context.Background(), 1)
  233. defer sem.Release(1)
  234. log := logrus.WithField("container", id)
  235. c, err := daemon.load(id)
  236. if err != nil {
  237. log.WithError(err).Error("failed to load container")
  238. return
  239. }
  240. if c.Driver != daemon.imageService.StorageDriver() {
  241. // Ignore the container if it wasn't created with the current storage-driver
  242. log.Debugf("not restoring container because it was created with another storage driver (%s)", c.Driver)
  243. return
  244. }
  245. if accessor, ok := daemon.imageService.(layerAccessor); ok {
  246. rwlayer, err := accessor.GetLayerByID(c.ID)
  247. if err != nil {
  248. log.WithError(err).Error("failed to load container mount")
  249. return
  250. }
  251. c.RWLayer = rwlayer
  252. }
  253. log.WithFields(logrus.Fields{
  254. "running": c.IsRunning(),
  255. "paused": c.IsPaused(),
  256. }).Debug("loaded container")
  257. mapLock.Lock()
  258. containers[c.ID] = c
  259. mapLock.Unlock()
  260. }(v.Name())
  261. }
  262. group.Wait()
  263. removeContainers := make(map[string]*container.Container)
  264. restartContainers := make(map[*container.Container]chan struct{})
  265. activeSandboxes := make(map[string]interface{})
  266. for _, c := range containers {
  267. group.Add(1)
  268. go func(c *container.Container) {
  269. defer group.Done()
  270. _ = sem.Acquire(context.Background(), 1)
  271. defer sem.Release(1)
  272. log := logrus.WithField("container", c.ID)
  273. if err := daemon.registerName(c); err != nil {
  274. log.WithError(err).Errorf("failed to register container name: %s", c.Name)
  275. mapLock.Lock()
  276. delete(containers, c.ID)
  277. mapLock.Unlock()
  278. return
  279. }
  280. if err := daemon.Register(c); err != nil {
  281. log.WithError(err).Error("failed to register container")
  282. mapLock.Lock()
  283. delete(containers, c.ID)
  284. mapLock.Unlock()
  285. return
  286. }
  287. }(c)
  288. }
  289. group.Wait()
  290. for _, c := range containers {
  291. group.Add(1)
  292. go func(c *container.Container) {
  293. defer group.Done()
  294. _ = sem.Acquire(context.Background(), 1)
  295. defer sem.Release(1)
  296. log := logrus.WithField("container", c.ID)
  297. if err := daemon.checkpointAndSave(c); err != nil {
  298. log.WithError(err).Error("error saving backported mountspec to disk")
  299. }
  300. daemon.setStateCounter(c)
  301. logger := func(c *container.Container) *logrus.Entry {
  302. return log.WithFields(logrus.Fields{
  303. "running": c.IsRunning(),
  304. "paused": c.IsPaused(),
  305. "restarting": c.IsRestarting(),
  306. })
  307. }
  308. logger(c).Debug("restoring container")
  309. var es *containerd.ExitStatus
  310. if err := c.RestoreTask(context.Background(), daemon.containerd); err != nil && !errdefs.IsNotFound(err) {
  311. logger(c).WithError(err).Error("failed to restore container with containerd")
  312. return
  313. }
  314. alive := false
  315. status := containerd.Unknown
  316. if tsk, ok := c.Task(); ok {
  317. s, err := tsk.Status(context.Background())
  318. if err != nil {
  319. logger(c).WithError(err).Error("failed to get task status")
  320. } else {
  321. status = s.Status
  322. alive = status != containerd.Stopped
  323. if !alive {
  324. logger(c).Debug("cleaning up dead container process")
  325. es, err = tsk.Delete(context.Background())
  326. if err != nil && !errdefs.IsNotFound(err) {
  327. logger(c).WithError(err).Error("failed to delete task from containerd")
  328. return
  329. }
  330. } else if !daemon.configStore.LiveRestoreEnabled {
  331. logger(c).Debug("shutting down container considered alive by containerd")
  332. if err := daemon.shutdownContainer(c); err != nil && !errdefs.IsNotFound(err) {
  333. log.WithError(err).Error("error shutting down container")
  334. return
  335. }
  336. status = containerd.Stopped
  337. alive = false
  338. c.ResetRestartManager(false)
  339. }
  340. }
  341. }
  342. // If the containerd task for the container was not found, docker's view of the
  343. // container state will be updated accordingly via SetStopped further down.
  344. if c.IsRunning() || c.IsPaused() {
  345. logger(c).Debug("syncing container on disk state with real state")
  346. c.RestartManager().Cancel() // manually start containers because some need to wait for swarm networking
  347. switch {
  348. case c.IsPaused() && alive:
  349. logger(c).WithField("state", status).Info("restored container paused")
  350. switch status {
  351. case containerd.Paused, containerd.Pausing:
  352. // nothing to do
  353. case containerd.Unknown, containerd.Stopped, "":
  354. log.WithField("status", status).Error("unexpected status for paused container during restore")
  355. default:
  356. // running
  357. c.Lock()
  358. c.Paused = false
  359. daemon.setStateCounter(c)
  360. daemon.updateHealthMonitor(c)
  361. if err := c.CheckpointTo(daemon.containersReplica); err != nil {
  362. log.WithError(err).Error("failed to update paused container state")
  363. }
  364. c.Unlock()
  365. }
  366. case !c.IsPaused() && alive:
  367. logger(c).Debug("restoring healthcheck")
  368. c.Lock()
  369. daemon.updateHealthMonitor(c)
  370. c.Unlock()
  371. }
  372. if !alive {
  373. logger(c).Debug("setting stopped state")
  374. c.Lock()
  375. var ces container.ExitStatus
  376. if es != nil {
  377. ces.ExitCode = int(es.ExitCode())
  378. ces.ExitedAt = es.ExitTime()
  379. }
  380. c.SetStopped(&ces)
  381. daemon.Cleanup(c)
  382. if err := c.CheckpointTo(daemon.containersReplica); err != nil {
  383. log.WithError(err).Error("failed to update stopped container state")
  384. }
  385. c.Unlock()
  386. logger(c).Debug("set stopped state")
  387. }
  388. // we call Mount and then Unmount to get BaseFs of the container
  389. if err := daemon.Mount(c); err != nil {
  390. // The mount is unlikely to fail. However, in case mount fails
  391. // the container should be allowed to restore here. Some functionalities
  392. // (like docker exec -u user) might be missing but container is able to be
  393. // stopped/restarted/removed.
  394. // See #29365 for related information.
  395. // The error is only logged here.
  396. logger(c).WithError(err).Warn("failed to mount container to get BaseFs path")
  397. } else {
  398. if err := daemon.Unmount(c); err != nil {
  399. logger(c).WithError(err).Warn("failed to umount container to get BaseFs path")
  400. }
  401. }
  402. c.ResetRestartManager(false)
  403. if !c.HostConfig.NetworkMode.IsContainer() && c.IsRunning() {
  404. options, err := daemon.buildSandboxOptions(c)
  405. if err != nil {
  406. logger(c).WithError(err).Warn("failed to build sandbox option to restore container")
  407. }
  408. mapLock.Lock()
  409. activeSandboxes[c.NetworkSettings.SandboxID] = options
  410. mapLock.Unlock()
  411. }
  412. }
  413. // get list of containers we need to restart
  414. // Do not autostart containers which
  415. // has endpoints in a swarm scope
  416. // network yet since the cluster is
  417. // not initialized yet. We will start
  418. // it after the cluster is
  419. // initialized.
  420. if daemon.configStore.AutoRestart && c.ShouldRestart() && !c.NetworkSettings.HasSwarmEndpoint && c.HasBeenStartedBefore {
  421. mapLock.Lock()
  422. restartContainers[c] = make(chan struct{})
  423. mapLock.Unlock()
  424. } else if c.HostConfig != nil && c.HostConfig.AutoRemove {
  425. mapLock.Lock()
  426. removeContainers[c.ID] = c
  427. mapLock.Unlock()
  428. }
  429. c.Lock()
  430. if c.RemovalInProgress {
  431. // We probably crashed in the middle of a removal, reset
  432. // the flag.
  433. //
  434. // We DO NOT remove the container here as we do not
  435. // know if the user had requested for either the
  436. // associated volumes, network links or both to also
  437. // be removed. So we put the container in the "dead"
  438. // state and leave further processing up to them.
  439. c.RemovalInProgress = false
  440. c.Dead = true
  441. if err := c.CheckpointTo(daemon.containersReplica); err != nil {
  442. log.WithError(err).Error("failed to update RemovalInProgress container state")
  443. } else {
  444. log.Debugf("reset RemovalInProgress state for container")
  445. }
  446. }
  447. c.Unlock()
  448. logger(c).Debug("done restoring container")
  449. }(c)
  450. }
  451. group.Wait()
  452. // Initialize the network controller and configure network settings.
  453. //
  454. // Note that we cannot initialize the network controller earlier, as it
  455. // needs to know if there's active sandboxes (running containers).
  456. if err = daemon.initNetworkController(activeSandboxes); err != nil {
  457. return fmt.Errorf("Error initializing network controller: %v", err)
  458. }
  459. // Now that all the containers are registered, register the links
  460. for _, c := range containers {
  461. group.Add(1)
  462. go func(c *container.Container) {
  463. _ = sem.Acquire(context.Background(), 1)
  464. if err := daemon.registerLinks(c, c.HostConfig); err != nil {
  465. logrus.WithField("container", c.ID).WithError(err).Error("failed to register link for container")
  466. }
  467. sem.Release(1)
  468. group.Done()
  469. }(c)
  470. }
  471. group.Wait()
  472. for c, notifier := range restartContainers {
  473. group.Add(1)
  474. go func(c *container.Container, chNotify chan struct{}) {
  475. _ = sem.Acquire(context.Background(), 1)
  476. log := logrus.WithField("container", c.ID)
  477. log.Debug("starting container")
  478. // ignore errors here as this is a best effort to wait for children to be
  479. // running before we try to start the container
  480. children := daemon.children(c)
  481. timeout := time.NewTimer(5 * time.Second)
  482. defer timeout.Stop()
  483. for _, child := range children {
  484. if notifier, exists := restartContainers[child]; exists {
  485. select {
  486. case <-notifier:
  487. case <-timeout.C:
  488. }
  489. }
  490. }
  491. if err := daemon.prepareMountPoints(c); err != nil {
  492. log.WithError(err).Error("failed to prepare mount points for container")
  493. }
  494. if err := daemon.containerStart(context.Background(), c, "", "", true); err != nil {
  495. log.WithError(err).Error("failed to start container")
  496. }
  497. close(chNotify)
  498. sem.Release(1)
  499. group.Done()
  500. }(c, notifier)
  501. }
  502. group.Wait()
  503. for id := range removeContainers {
  504. group.Add(1)
  505. go func(cid string) {
  506. _ = sem.Acquire(context.Background(), 1)
  507. if err := daemon.ContainerRm(cid, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil {
  508. logrus.WithField("container", cid).WithError(err).Error("failed to remove container")
  509. }
  510. sem.Release(1)
  511. group.Done()
  512. }(id)
  513. }
  514. group.Wait()
  515. // any containers that were started above would already have had this done,
  516. // however we need to now prepare the mountpoints for the rest of the containers as well.
  517. // This shouldn't cause any issue running on the containers that already had this run.
  518. // This must be run after any containers with a restart policy so that containerized plugins
  519. // can have a chance to be running before we try to initialize them.
  520. for _, c := range containers {
  521. // if the container has restart policy, do not
  522. // prepare the mountpoints since it has been done on restarting.
  523. // This is to speed up the daemon start when a restart container
  524. // has a volume and the volume driver is not available.
  525. if _, ok := restartContainers[c]; ok {
  526. continue
  527. } else if _, ok := removeContainers[c.ID]; ok {
  528. // container is automatically removed, skip it.
  529. continue
  530. }
  531. group.Add(1)
  532. go func(c *container.Container) {
  533. _ = sem.Acquire(context.Background(), 1)
  534. if err := daemon.prepareMountPoints(c); err != nil {
  535. logrus.WithField("container", c.ID).WithError(err).Error("failed to prepare mountpoints for container")
  536. }
  537. sem.Release(1)
  538. group.Done()
  539. }(c)
  540. }
  541. group.Wait()
  542. logrus.Info("Loading containers: done.")
  543. return nil
  544. }
  545. // RestartSwarmContainers restarts any autostart container which has a
  546. // swarm endpoint.
  547. func (daemon *Daemon) RestartSwarmContainers() {
  548. ctx := context.Background()
  549. // parallelLimit is the maximum number of parallel startup jobs that we
  550. // allow (this is the limited used for all startup semaphores). The multipler
  551. // (128) was chosen after some fairly significant benchmarking -- don't change
  552. // it unless you've tested it significantly (this value is adjusted if
  553. // RLIMIT_NOFILE is small to avoid EMFILE).
  554. parallelLimit := adjustParallelLimit(len(daemon.List()), 128*runtime.NumCPU())
  555. var group sync.WaitGroup
  556. sem := semaphore.NewWeighted(int64(parallelLimit))
  557. for _, c := range daemon.List() {
  558. if !c.IsRunning() && !c.IsPaused() {
  559. // Autostart all the containers which has a
  560. // swarm endpoint now that the cluster is
  561. // initialized.
  562. if daemon.configStore.AutoRestart && c.ShouldRestart() && c.NetworkSettings.HasSwarmEndpoint && c.HasBeenStartedBefore {
  563. group.Add(1)
  564. go func(c *container.Container) {
  565. if err := sem.Acquire(ctx, 1); err != nil {
  566. // ctx is done.
  567. group.Done()
  568. return
  569. }
  570. if err := daemon.containerStart(ctx, c, "", "", true); err != nil {
  571. logrus.WithField("container", c.ID).WithError(err).Error("failed to start swarm container")
  572. }
  573. sem.Release(1)
  574. group.Done()
  575. }(c)
  576. }
  577. }
  578. }
  579. group.Wait()
  580. }
  581. func (daemon *Daemon) children(c *container.Container) map[string]*container.Container {
  582. return daemon.linkIndex.children(c)
  583. }
  584. // parents returns the names of the parent containers of the container
  585. // with the given name.
  586. func (daemon *Daemon) parents(c *container.Container) map[string]*container.Container {
  587. return daemon.linkIndex.parents(c)
  588. }
  589. func (daemon *Daemon) registerLink(parent, child *container.Container, alias string) error {
  590. fullName := path.Join(parent.Name, alias)
  591. if err := daemon.containersReplica.ReserveName(fullName, child.ID); err != nil {
  592. if errors.Is(err, container.ErrNameReserved) {
  593. logrus.Warnf("error registering link for %s, to %s, as alias %s, ignoring: %v", parent.ID, child.ID, alias, err)
  594. return nil
  595. }
  596. return err
  597. }
  598. daemon.linkIndex.link(parent, child, fullName)
  599. return nil
  600. }
  601. // DaemonJoinsCluster informs the daemon has joined the cluster and provides
  602. // the handler to query the cluster component
  603. func (daemon *Daemon) DaemonJoinsCluster(clusterProvider cluster.Provider) {
  604. daemon.setClusterProvider(clusterProvider)
  605. }
  606. // DaemonLeavesCluster informs the daemon has left the cluster
  607. func (daemon *Daemon) DaemonLeavesCluster() {
  608. // Daemon is in charge of removing the attachable networks with
  609. // connected containers when the node leaves the swarm
  610. daemon.clearAttachableNetworks()
  611. // We no longer need the cluster provider, stop it now so that
  612. // the network agent will stop listening to cluster events.
  613. daemon.setClusterProvider(nil)
  614. // Wait for the networking cluster agent to stop
  615. daemon.netController.AgentStopWait()
  616. // Daemon is in charge of removing the ingress network when the
  617. // node leaves the swarm. Wait for job to be done or timeout.
  618. // This is called also on graceful daemon shutdown. We need to
  619. // wait, because the ingress release has to happen before the
  620. // network controller is stopped.
  621. if done, err := daemon.ReleaseIngress(); err == nil {
  622. timeout := time.NewTimer(5 * time.Second)
  623. defer timeout.Stop()
  624. select {
  625. case <-done:
  626. case <-timeout.C:
  627. logrus.Warn("timeout while waiting for ingress network removal")
  628. }
  629. } else {
  630. logrus.Warnf("failed to initiate ingress network removal: %v", err)
  631. }
  632. daemon.attachmentStore.ClearAttachments()
  633. }
  634. // setClusterProvider sets a component for querying the current cluster state.
  635. func (daemon *Daemon) setClusterProvider(clusterProvider cluster.Provider) {
  636. daemon.clusterProvider = clusterProvider
  637. daemon.netController.SetClusterProvider(clusterProvider)
  638. daemon.attachableNetworkLock = locker.New()
  639. }
  640. // IsSwarmCompatible verifies if the current daemon
  641. // configuration is compatible with the swarm mode
  642. func (daemon *Daemon) IsSwarmCompatible() error {
  643. if daemon.configStore == nil {
  644. return nil
  645. }
  646. return daemon.configStore.IsSwarmCompatible()
  647. }
  648. // NewDaemon sets up everything for the daemon to be able to service
  649. // requests from the webserver.
  650. func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.Store, authzMiddleware *authorization.Middleware) (daemon *Daemon, err error) {
  651. // Verify platform-specific requirements.
  652. // TODO(thaJeztah): this should be called before we try to create the daemon; perhaps together with the config validation.
  653. if err := checkSystem(); err != nil {
  654. return nil, err
  655. }
  656. registryService, err := registry.NewService(config.ServiceOptions)
  657. if err != nil {
  658. return nil, err
  659. }
  660. // Ensure that we have a correct root key limit for launching containers.
  661. if err := modifyRootKeyLimit(); err != nil {
  662. logrus.Warnf("unable to modify root key limit, number of containers could be limited by this quota: %v", err)
  663. }
  664. // Ensure we have compatible and valid configuration options
  665. if err := verifyDaemonSettings(config); err != nil {
  666. return nil, err
  667. }
  668. // Do we have a disabled network?
  669. config.DisableBridge = isBridgeNetworkDisabled(config)
  670. // Setup the resolv.conf
  671. setupResolvConf(config)
  672. idMapping, err := setupRemappedRoot(config)
  673. if err != nil {
  674. return nil, err
  675. }
  676. rootIDs := idMapping.RootPair()
  677. if err := setupDaemonProcess(config); err != nil {
  678. return nil, err
  679. }
  680. // set up the tmpDir to use a canonical path
  681. tmp, err := prepareTempDir(config.Root)
  682. if err != nil {
  683. return nil, fmt.Errorf("Unable to get the TempDir under %s: %s", config.Root, err)
  684. }
  685. realTmp, err := fileutils.ReadSymlinkedDirectory(tmp)
  686. if err != nil {
  687. return nil, fmt.Errorf("Unable to get the full path to the TempDir (%s): %s", tmp, err)
  688. }
  689. if isWindows {
  690. if err := system.MkdirAll(realTmp, 0); err != nil {
  691. return nil, fmt.Errorf("Unable to create the TempDir (%s): %s", realTmp, err)
  692. }
  693. os.Setenv("TEMP", realTmp)
  694. os.Setenv("TMP", realTmp)
  695. } else {
  696. os.Setenv("TMPDIR", realTmp)
  697. }
  698. d := &Daemon{
  699. configStore: config,
  700. PluginStore: pluginStore,
  701. startupDone: make(chan struct{}),
  702. }
  703. // Ensure the daemon is properly shutdown if there is a failure during
  704. // initialization
  705. defer func() {
  706. if err != nil {
  707. // Use a fresh context here. Passed context could be cancelled.
  708. if err := d.Shutdown(context.Background()); err != nil {
  709. logrus.Error(err)
  710. }
  711. }
  712. }()
  713. if err := d.setGenericResources(config); err != nil {
  714. return nil, err
  715. }
  716. // set up SIGUSR1 handler on Unix-like systems, or a Win32 global event
  717. // on Windows to dump Go routine stacks
  718. stackDumpDir := config.Root
  719. if execRoot := config.GetExecRoot(); execRoot != "" {
  720. stackDumpDir = execRoot
  721. }
  722. d.setupDumpStackTrap(stackDumpDir)
  723. if err := d.setupSeccompProfile(); err != nil {
  724. return nil, err
  725. }
  726. // Set the default isolation mode (only applicable on Windows)
  727. if err := d.setDefaultIsolation(); err != nil {
  728. return nil, fmt.Errorf("error setting default isolation mode: %v", err)
  729. }
  730. if err := configureMaxThreads(config); err != nil {
  731. logrus.Warnf("Failed to configure golang's threads limit: %v", err)
  732. }
  733. // ensureDefaultAppArmorProfile does nothing if apparmor is disabled
  734. if err := ensureDefaultAppArmorProfile(); err != nil {
  735. logrus.Errorf(err.Error())
  736. }
  737. daemonRepo := filepath.Join(config.Root, "containers")
  738. if err := idtools.MkdirAllAndChown(daemonRepo, 0o710, idtools.Identity{
  739. UID: idtools.CurrentIdentity().UID,
  740. GID: rootIDs.GID,
  741. }); err != nil {
  742. return nil, err
  743. }
  744. // Create the directory where we'll store the runtime scripts (i.e. in
  745. // order to support runtimeArgs)
  746. if err = os.Mkdir(filepath.Join(config.Root, "runtimes"), 0o700); err != nil && !errors.Is(err, os.ErrExist) {
  747. return nil, err
  748. }
  749. if err := d.loadRuntimes(); err != nil {
  750. return nil, err
  751. }
  752. if isWindows {
  753. // Note that permissions (0o700) are ignored on Windows; passing them to
  754. // show intent only. We could consider using idtools.MkdirAndChown here
  755. // to apply an ACL.
  756. if err = os.Mkdir(filepath.Join(config.Root, "credentialspecs"), 0o700); err != nil && !errors.Is(err, os.ErrExist) {
  757. return nil, err
  758. }
  759. }
  760. d.registryService = registryService
  761. dlogger.RegisterPluginGetter(d.PluginStore)
  762. metricsSockPath, err := d.listenMetricsSock()
  763. if err != nil {
  764. return nil, err
  765. }
  766. registerMetricsPluginCallback(d.PluginStore, metricsSockPath)
  767. backoffConfig := backoff.DefaultConfig
  768. backoffConfig.MaxDelay = 3 * time.Second
  769. connParams := grpc.ConnectParams{
  770. Backoff: backoffConfig,
  771. }
  772. gopts := []grpc.DialOption{
  773. // WithBlock makes sure that the following containerd request
  774. // is reliable.
  775. //
  776. // NOTE: In one edge case with high load pressure, kernel kills
  777. // dockerd, containerd and containerd-shims caused by OOM.
  778. // When both dockerd and containerd restart, but containerd
  779. // will take time to recover all the existing containers. Before
  780. // containerd serving, dockerd will failed with gRPC error.
  781. // That bad thing is that restore action will still ignore the
  782. // any non-NotFound errors and returns running state for
  783. // already stopped container. It is unexpected behavior. And
  784. // we need to restart dockerd to make sure that anything is OK.
  785. //
  786. // It is painful. Add WithBlock can prevent the edge case. And
  787. // n common case, the containerd will be serving in shortly.
  788. // It is not harm to add WithBlock for containerd connection.
  789. grpc.WithBlock(),
  790. grpc.WithTransportCredentials(insecure.NewCredentials()),
  791. grpc.WithConnectParams(connParams),
  792. grpc.WithContextDialer(dialer.ContextDialer),
  793. // TODO(stevvooe): We may need to allow configuration of this on the client.
  794. grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
  795. grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
  796. }
  797. if config.ContainerdAddr != "" {
  798. d.containerdCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(config.ContainerdNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
  799. if err != nil {
  800. return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
  801. }
  802. }
  803. createPluginExec := func(m *plugin.Manager) (plugin.Executor, error) {
  804. var pluginCli *containerd.Client
  805. if config.ContainerdAddr != "" {
  806. pluginCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(config.ContainerdPluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
  807. if err != nil {
  808. return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
  809. }
  810. }
  811. var (
  812. shim string
  813. shimOpts interface{}
  814. )
  815. if runtime.GOOS != "windows" {
  816. shim, shimOpts, err = d.getRuntime(config.GetDefaultRuntimeName())
  817. if err != nil {
  818. return nil, err
  819. }
  820. }
  821. return pluginexec.New(ctx, getPluginExecRoot(config), pluginCli, config.ContainerdPluginNamespace, m, shim, shimOpts)
  822. }
  823. // Plugin system initialization should happen before restore. Do not change order.
  824. d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{
  825. Root: filepath.Join(config.Root, "plugins"),
  826. ExecRoot: getPluginExecRoot(config),
  827. Store: d.PluginStore,
  828. CreateExecutor: createPluginExec,
  829. RegistryService: registryService,
  830. LiveRestoreEnabled: config.LiveRestoreEnabled,
  831. LogPluginEvent: d.LogPluginEvent, // todo: make private
  832. AuthzMiddleware: authzMiddleware,
  833. })
  834. if err != nil {
  835. return nil, errors.Wrap(err, "couldn't create plugin manager")
  836. }
  837. if err := d.setupDefaultLogConfig(); err != nil {
  838. return nil, err
  839. }
  840. d.volumes, err = volumesservice.NewVolumeService(config.Root, d.PluginStore, rootIDs, d)
  841. if err != nil {
  842. return nil, err
  843. }
  844. // Check if Devices cgroup is mounted, it is hard requirement for container security,
  845. // on Linux.
  846. //
  847. // Important: we call getSysInfo() directly here, without storing the results,
  848. // as networking has not yet been set up, so we only have partial system info
  849. // at this point.
  850. //
  851. // TODO(thaJeztah) add a utility to only collect the CgroupDevicesEnabled information
  852. if runtime.GOOS == "linux" && !userns.RunningInUserNS() && !getSysInfo(d).CgroupDevicesEnabled {
  853. return nil, errors.New("Devices cgroup isn't mounted")
  854. }
  855. d.id, err = loadOrCreateID(filepath.Join(config.Root, "engine-id"))
  856. if err != nil {
  857. return nil, err
  858. }
  859. d.repository = daemonRepo
  860. d.containers = container.NewMemoryStore()
  861. if d.containersReplica, err = container.NewViewDB(); err != nil {
  862. return nil, err
  863. }
  864. d.execCommands = container.NewExecStore()
  865. d.statsCollector = d.newStatsCollector(1 * time.Second)
  866. d.EventsService = events.New()
  867. d.root = config.Root
  868. d.idMapping = idMapping
  869. d.linkIndex = newLinkIndex()
  870. // On Windows we don't support the environment variable, or a user supplied graphdriver
  871. // Unix platforms however run a single graphdriver for all containers, and it can
  872. // be set through an environment variable, a daemon start parameter, or chosen through
  873. // initialization of the layerstore through driver priority order for example.
  874. driverName := os.Getenv("DOCKER_DRIVER")
  875. if isWindows {
  876. driverName = "windowsfilter"
  877. } else if driverName != "" {
  878. logrus.Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", driverName)
  879. } else {
  880. driverName = config.GraphDriver
  881. }
  882. if d.UsesSnapshotter() {
  883. if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" {
  884. logrus.Warn("Enabling containerd snapshotter through the $TEST_INTEGRATION_USE_SNAPSHOTTER environment variable. This should only be used for testing.")
  885. }
  886. logrus.Info("Starting daemon with containerd snapshotter integration enabled")
  887. // FIXME(thaJeztah): implement automatic snapshotter-selection similar to graph-driver selection; see https://github.com/moby/moby/issues/44076
  888. if driverName == "" {
  889. driverName = containerd.DefaultSnapshotter
  890. }
  891. // Configure and validate the kernels security support. Note this is a Linux/FreeBSD
  892. // operation only, so it is safe to pass *just* the runtime OS graphdriver.
  893. if err := configureKernelSecuritySupport(config, driverName); err != nil {
  894. return nil, err
  895. }
  896. d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{
  897. Client: d.containerdCli,
  898. Containers: d.containers,
  899. Snapshotter: driverName,
  900. HostsProvider: d,
  901. Registry: d.registryService,
  902. EventsService: d.EventsService,
  903. })
  904. } else {
  905. layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{
  906. Root: config.Root,
  907. MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),
  908. GraphDriver: driverName,
  909. GraphDriverOptions: config.GraphOptions,
  910. IDMapping: idMapping,
  911. PluginGetter: d.PluginStore,
  912. ExperimentalEnabled: config.Experimental,
  913. })
  914. if err != nil {
  915. return nil, err
  916. }
  917. // Configure and validate the kernels security support. Note this is a Linux/FreeBSD
  918. // operation only, so it is safe to pass *just* the runtime OS graphdriver.
  919. if err := configureKernelSecuritySupport(config, layerStore.DriverName()); err != nil {
  920. return nil, err
  921. }
  922. imageRoot := filepath.Join(config.Root, "image", layerStore.DriverName())
  923. ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))
  924. if err != nil {
  925. return nil, err
  926. }
  927. // We have a single tag/reference store for the daemon globally. However, it's
  928. // stored under the graphdriver. On host platforms which only support a single
  929. // container OS, but multiple selectable graphdrivers, this means depending on which
  930. // graphdriver is chosen, the global reference store is under there. For
  931. // platforms which support multiple container operating systems, this is slightly
  932. // more problematic as where does the global ref store get located? Fortunately,
  933. // for Windows, which is currently the only daemon supporting multiple container
  934. // operating systems, the list of graphdrivers available isn't user configurable.
  935. // For backwards compatibility, we just put it under the windowsfilter
  936. // directory regardless.
  937. refStoreLocation := filepath.Join(imageRoot, `repositories.json`)
  938. rs, err := refstore.NewReferenceStore(refStoreLocation)
  939. if err != nil {
  940. return nil, fmt.Errorf("Couldn't create reference store repository: %s", err)
  941. }
  942. d.ReferenceStore = rs
  943. imageStore, err := image.NewImageStore(ifs, layerStore)
  944. if err != nil {
  945. return nil, err
  946. }
  947. distributionMetadataStore, err := dmetadata.NewFSMetadataStore(filepath.Join(imageRoot, "distribution"))
  948. if err != nil {
  949. return nil, err
  950. }
  951. imgSvcConfig := images.ImageServiceConfig{
  952. ContainerStore: d.containers,
  953. DistributionMetadataStore: distributionMetadataStore,
  954. EventsService: d.EventsService,
  955. ImageStore: imageStore,
  956. LayerStore: layerStore,
  957. MaxConcurrentDownloads: config.MaxConcurrentDownloads,
  958. MaxConcurrentUploads: config.MaxConcurrentUploads,
  959. MaxDownloadAttempts: config.MaxDownloadAttempts,
  960. ReferenceStore: rs,
  961. RegistryService: registryService,
  962. ContentNamespace: config.ContainerdNamespace,
  963. }
  964. // containerd is not currently supported with Windows.
  965. // So sometimes d.containerdCli will be nil
  966. // In that case we'll create a local content store... but otherwise we'll use containerd
  967. if d.containerdCli != nil {
  968. imgSvcConfig.Leases = d.containerdCli.LeasesService()
  969. imgSvcConfig.ContentStore = d.containerdCli.ContentStore()
  970. } else {
  971. cs, lm, err := d.configureLocalContentStore(config.ContainerdNamespace)
  972. if err != nil {
  973. return nil, err
  974. }
  975. imgSvcConfig.ContentStore = cs
  976. imgSvcConfig.Leases = lm
  977. }
  978. // TODO: imageStore, distributionMetadataStore, and ReferenceStore are only
  979. // used above to run migration. They could be initialized in ImageService
  980. // if migration is called from daemon/images. layerStore might move as well.
  981. d.imageService = images.NewImageService(imgSvcConfig)
  982. logrus.Debugf("Max Concurrent Downloads: %d", imgSvcConfig.MaxConcurrentDownloads)
  983. logrus.Debugf("Max Concurrent Uploads: %d", imgSvcConfig.MaxConcurrentUploads)
  984. logrus.Debugf("Max Download Attempts: %d", imgSvcConfig.MaxDownloadAttempts)
  985. }
  986. go d.execCommandGC()
  987. if err := d.initLibcontainerd(ctx); err != nil {
  988. return nil, err
  989. }
  990. if err := d.restore(); err != nil {
  991. return nil, err
  992. }
  993. close(d.startupDone)
  994. info := d.SystemInfo()
  995. for _, w := range info.Warnings {
  996. logrus.Warn(w)
  997. }
  998. engineInfo.WithValues(
  999. dockerversion.Version,
  1000. dockerversion.GitCommit,
  1001. info.Architecture,
  1002. info.Driver,
  1003. info.KernelVersion,
  1004. info.OperatingSystem,
  1005. info.OSType,
  1006. info.OSVersion,
  1007. info.ID,
  1008. ).Set(1)
  1009. engineCpus.Set(float64(info.NCPU))
  1010. engineMemory.Set(float64(info.MemTotal))
  1011. logrus.WithFields(logrus.Fields{
  1012. "version": dockerversion.Version,
  1013. "commit": dockerversion.GitCommit,
  1014. "graphdriver": d.ImageService().StorageDriver(),
  1015. }).Info("Docker daemon")
  1016. return d, nil
  1017. }
  1018. // DistributionServices returns services controlling daemon storage
  1019. func (daemon *Daemon) DistributionServices() images.DistributionServices {
  1020. return daemon.imageService.DistributionServices()
  1021. }
  1022. func (daemon *Daemon) waitForStartupDone() {
  1023. <-daemon.startupDone
  1024. }
  1025. func (daemon *Daemon) shutdownContainer(c *container.Container) error {
  1026. // If container failed to exit in stopTimeout seconds of SIGTERM, then using the force
  1027. if err := daemon.containerStop(context.TODO(), c, containertypes.StopOptions{}); err != nil {
  1028. return fmt.Errorf("Failed to stop container %s with error: %v", c.ID, err)
  1029. }
  1030. // Wait without timeout for the container to exit.
  1031. // Ignore the result.
  1032. <-c.Wait(context.Background(), container.WaitConditionNotRunning)
  1033. return nil
  1034. }
  1035. // ShutdownTimeout returns the timeout (in seconds) before containers are forcibly
  1036. // killed during shutdown. The default timeout can be configured both on the daemon
  1037. // and per container, and the longest timeout will be used. A grace-period of
  1038. // 5 seconds is added to the configured timeout.
  1039. //
  1040. // A negative (-1) timeout means "indefinitely", which means that containers
  1041. // are not forcibly killed, and the daemon shuts down after all containers exit.
  1042. func (daemon *Daemon) ShutdownTimeout() int {
  1043. shutdownTimeout := daemon.configStore.ShutdownTimeout
  1044. if shutdownTimeout < 0 {
  1045. return -1
  1046. }
  1047. if daemon.containers == nil {
  1048. return shutdownTimeout
  1049. }
  1050. graceTimeout := 5
  1051. for _, c := range daemon.containers.List() {
  1052. stopTimeout := c.StopTimeout()
  1053. if stopTimeout < 0 {
  1054. return -1
  1055. }
  1056. if stopTimeout+graceTimeout > shutdownTimeout {
  1057. shutdownTimeout = stopTimeout + graceTimeout
  1058. }
  1059. }
  1060. return shutdownTimeout
  1061. }
  1062. // Shutdown stops the daemon.
  1063. func (daemon *Daemon) Shutdown(ctx context.Context) error {
  1064. daemon.shutdown = true
  1065. // Keep mounts and networking running on daemon shutdown if
  1066. // we are to keep containers running and restore them.
  1067. if daemon.configStore.LiveRestoreEnabled && daemon.containers != nil {
  1068. // check if there are any running containers, if none we should do some cleanup
  1069. if ls, err := daemon.Containers(ctx, &types.ContainerListOptions{}); len(ls) != 0 || err != nil {
  1070. // metrics plugins still need some cleanup
  1071. daemon.cleanupMetricsPlugins()
  1072. return err
  1073. }
  1074. }
  1075. if daemon.containers != nil {
  1076. logrus.Debugf("daemon configured with a %d seconds minimum shutdown timeout", daemon.configStore.ShutdownTimeout)
  1077. logrus.Debugf("start clean shutdown of all containers with a %d seconds timeout...", daemon.ShutdownTimeout())
  1078. daemon.containers.ApplyAll(func(c *container.Container) {
  1079. if !c.IsRunning() {
  1080. return
  1081. }
  1082. log := logrus.WithField("container", c.ID)
  1083. log.Debug("shutting down container")
  1084. if err := daemon.shutdownContainer(c); err != nil {
  1085. log.WithError(err).Error("failed to shut down container")
  1086. return
  1087. }
  1088. if mountid, err := daemon.imageService.GetLayerMountID(c.ID); err == nil {
  1089. daemon.cleanupMountsByID(mountid)
  1090. }
  1091. log.Debugf("shut down container")
  1092. })
  1093. }
  1094. if daemon.volumes != nil {
  1095. if err := daemon.volumes.Shutdown(); err != nil {
  1096. logrus.Errorf("Error shutting down volume store: %v", err)
  1097. }
  1098. }
  1099. if daemon.imageService != nil {
  1100. if err := daemon.imageService.Cleanup(); err != nil {
  1101. logrus.Error(err)
  1102. }
  1103. }
  1104. // If we are part of a cluster, clean up cluster's stuff
  1105. if daemon.clusterProvider != nil {
  1106. logrus.Debugf("start clean shutdown of cluster resources...")
  1107. daemon.DaemonLeavesCluster()
  1108. }
  1109. daemon.cleanupMetricsPlugins()
  1110. // Shutdown plugins after containers and layerstore. Don't change the order.
  1111. daemon.pluginShutdown()
  1112. // trigger libnetwork Stop only if it's initialized
  1113. if daemon.netController != nil {
  1114. daemon.netController.Stop()
  1115. }
  1116. if daemon.containerdCli != nil {
  1117. daemon.containerdCli.Close()
  1118. }
  1119. if daemon.mdDB != nil {
  1120. daemon.mdDB.Close()
  1121. }
  1122. return daemon.cleanupMounts()
  1123. }
  1124. // Mount sets container.BaseFS
  1125. func (daemon *Daemon) Mount(container *container.Container) error {
  1126. return daemon.imageService.Mount(context.Background(), container)
  1127. }
  1128. // Unmount unsets the container base filesystem
  1129. func (daemon *Daemon) Unmount(container *container.Container) error {
  1130. return daemon.imageService.Unmount(context.Background(), container)
  1131. }
  1132. // Subnets return the IPv4 and IPv6 subnets of networks that are manager by Docker.
  1133. func (daemon *Daemon) Subnets() ([]net.IPNet, []net.IPNet) {
  1134. var v4Subnets []net.IPNet
  1135. var v6Subnets []net.IPNet
  1136. managedNetworks := daemon.netController.Networks()
  1137. for _, managedNetwork := range managedNetworks {
  1138. v4infos, v6infos := managedNetwork.Info().IpamInfo()
  1139. for _, info := range v4infos {
  1140. if info.IPAMData.Pool != nil {
  1141. v4Subnets = append(v4Subnets, *info.IPAMData.Pool)
  1142. }
  1143. }
  1144. for _, info := range v6infos {
  1145. if info.IPAMData.Pool != nil {
  1146. v6Subnets = append(v6Subnets, *info.IPAMData.Pool)
  1147. }
  1148. }
  1149. }
  1150. return v4Subnets, v6Subnets
  1151. }
  1152. // prepareTempDir prepares and returns the default directory to use
  1153. // for temporary files.
  1154. // If it doesn't exist, it is created. If it exists, its content is removed.
  1155. func prepareTempDir(rootDir string) (string, error) {
  1156. var tmpDir string
  1157. if tmpDir = os.Getenv("DOCKER_TMPDIR"); tmpDir == "" {
  1158. tmpDir = filepath.Join(rootDir, "tmp")
  1159. newName := tmpDir + "-old"
  1160. if err := os.Rename(tmpDir, newName); err == nil {
  1161. go func() {
  1162. if err := os.RemoveAll(newName); err != nil {
  1163. logrus.Warnf("failed to delete old tmp directory: %s", newName)
  1164. }
  1165. }()
  1166. } else if !os.IsNotExist(err) {
  1167. logrus.Warnf("failed to rename %s for background deletion: %s. Deleting synchronously", tmpDir, err)
  1168. if err := os.RemoveAll(tmpDir); err != nil {
  1169. logrus.Warnf("failed to delete old tmp directory: %s", tmpDir)
  1170. }
  1171. }
  1172. }
  1173. return tmpDir, idtools.MkdirAllAndChown(tmpDir, 0o700, idtools.CurrentIdentity())
  1174. }
  1175. func (daemon *Daemon) setGenericResources(conf *config.Config) error {
  1176. genericResources, err := config.ParseGenericResources(conf.NodeGenericResources)
  1177. if err != nil {
  1178. return err
  1179. }
  1180. daemon.genericResources = genericResources
  1181. return nil
  1182. }
  1183. // IsShuttingDown tells whether the daemon is shutting down or not
  1184. func (daemon *Daemon) IsShuttingDown() bool {
  1185. return daemon.shutdown
  1186. }
  1187. func isBridgeNetworkDisabled(conf *config.Config) bool {
  1188. return conf.BridgeConfig.Iface == config.DisableNetworkBridge
  1189. }
  1190. func (daemon *Daemon) networkOptions(pg plugingetter.PluginGetter, activeSandboxes map[string]interface{}) ([]nwconfig.Option, error) {
  1191. options := []nwconfig.Option{}
  1192. if daemon.configStore == nil {
  1193. return options, nil
  1194. }
  1195. conf := daemon.configStore
  1196. dd := runconfig.DefaultDaemonNetworkMode()
  1197. options = []nwconfig.Option{
  1198. nwconfig.OptionDataDir(conf.Root),
  1199. nwconfig.OptionExecRoot(conf.GetExecRoot()),
  1200. nwconfig.OptionDefaultDriver(string(dd)),
  1201. nwconfig.OptionDefaultNetwork(dd.NetworkName()),
  1202. nwconfig.OptionLabels(conf.Labels),
  1203. nwconfig.OptionNetworkControlPlaneMTU(conf.NetworkControlPlaneMTU),
  1204. driverOptions(conf),
  1205. }
  1206. if len(conf.NetworkConfig.DefaultAddressPools.Value()) > 0 {
  1207. options = append(options, nwconfig.OptionDefaultAddressPoolConfig(conf.NetworkConfig.DefaultAddressPools.Value()))
  1208. }
  1209. if conf.LiveRestoreEnabled && len(activeSandboxes) != 0 {
  1210. options = append(options, nwconfig.OptionActiveSandboxes(activeSandboxes))
  1211. }
  1212. if pg != nil {
  1213. options = append(options, nwconfig.OptionPluginGetter(pg))
  1214. }
  1215. return options, nil
  1216. }
  1217. // GetCluster returns the cluster
  1218. func (daemon *Daemon) GetCluster() Cluster {
  1219. return daemon.cluster
  1220. }
  1221. // SetCluster sets the cluster
  1222. func (daemon *Daemon) SetCluster(cluster Cluster) {
  1223. daemon.cluster = cluster
  1224. }
  1225. func (daemon *Daemon) pluginShutdown() {
  1226. manager := daemon.pluginManager
  1227. // Check for a valid manager object. In error conditions, daemon init can fail
  1228. // and shutdown called, before plugin manager is initialized.
  1229. if manager != nil {
  1230. manager.Shutdown()
  1231. }
  1232. }
  1233. // PluginManager returns current pluginManager associated with the daemon
  1234. func (daemon *Daemon) PluginManager() *plugin.Manager { // set up before daemon to avoid this method
  1235. return daemon.pluginManager
  1236. }
  1237. // PluginGetter returns current pluginStore associated with the daemon
  1238. func (daemon *Daemon) PluginGetter() *plugin.Store {
  1239. return daemon.PluginStore
  1240. }
  1241. // CreateDaemonRoot creates the root for the daemon
  1242. func CreateDaemonRoot(config *config.Config) error {
  1243. // get the canonical path to the Docker root directory
  1244. var realRoot string
  1245. if _, err := os.Stat(config.Root); err != nil && os.IsNotExist(err) {
  1246. realRoot = config.Root
  1247. } else {
  1248. realRoot, err = fileutils.ReadSymlinkedDirectory(config.Root)
  1249. if err != nil {
  1250. return fmt.Errorf("Unable to get the full path to root (%s): %s", config.Root, err)
  1251. }
  1252. }
  1253. idMapping, err := setupRemappedRoot(config)
  1254. if err != nil {
  1255. return err
  1256. }
  1257. return setupDaemonRoot(config, realRoot, idMapping.RootPair())
  1258. }
  1259. // checkpointAndSave grabs a container lock to safely call container.CheckpointTo
  1260. func (daemon *Daemon) checkpointAndSave(container *container.Container) error {
  1261. container.Lock()
  1262. defer container.Unlock()
  1263. if err := container.CheckpointTo(daemon.containersReplica); err != nil {
  1264. return fmt.Errorf("Error saving container state: %v", err)
  1265. }
  1266. return nil
  1267. }
  1268. // because the CLI sends a -1 when it wants to unset the swappiness value
  1269. // we need to clear it on the server side
  1270. func fixMemorySwappiness(resources *containertypes.Resources) {
  1271. if resources.MemorySwappiness != nil && *resources.MemorySwappiness == -1 {
  1272. resources.MemorySwappiness = nil
  1273. }
  1274. }
  1275. // GetAttachmentStore returns current attachment store associated with the daemon
  1276. func (daemon *Daemon) GetAttachmentStore() *network.AttachmentStore {
  1277. return &daemon.attachmentStore
  1278. }
  1279. // IdentityMapping returns uid/gid mapping or a SID (in the case of Windows) for the builder
  1280. func (daemon *Daemon) IdentityMapping() idtools.IdentityMapping {
  1281. return daemon.idMapping
  1282. }
  1283. // ImageService returns the Daemon's ImageService
  1284. func (daemon *Daemon) ImageService() ImageService {
  1285. return daemon.imageService
  1286. }
  1287. // ImageBackend returns an image-backend for Swarm and the distribution router.
  1288. func (daemon *Daemon) ImageBackend() executorpkg.ImageBackend {
  1289. return &imageBackend{
  1290. ImageService: daemon.imageService,
  1291. registryService: daemon.registryService,
  1292. }
  1293. }
  1294. // RegistryService returns the Daemon's RegistryService
  1295. func (daemon *Daemon) RegistryService() *registry.Service {
  1296. return daemon.registryService
  1297. }
  1298. // BuilderBackend returns the backend used by builder
  1299. func (daemon *Daemon) BuilderBackend() builder.Backend {
  1300. return struct {
  1301. *Daemon
  1302. ImageService
  1303. }{daemon, daemon.imageService}
  1304. }
  1305. // RawSysInfo returns *sysinfo.SysInfo .
  1306. func (daemon *Daemon) RawSysInfo() *sysinfo.SysInfo {
  1307. daemon.sysInfoOnce.Do(func() {
  1308. // We check if sysInfo is not set here, to allow some test to
  1309. // override the actual sysInfo.
  1310. if daemon.sysInfo == nil {
  1311. daemon.sysInfo = getSysInfo(daemon)
  1312. }
  1313. })
  1314. return daemon.sysInfo
  1315. }
  1316. // imageBackend is used to satisfy the [executorpkg.ImageBackend] and
  1317. // [github.com/docker/docker/api/server/router/distribution.Backend]
  1318. // interfaces.
  1319. type imageBackend struct {
  1320. ImageService
  1321. registryService *registry.Service
  1322. }
  1323. // GetRepository returns a repository from the registry.
  1324. func (i *imageBackend) GetRepository(ctx context.Context, ref reference.Named, authConfig *registrytypes.AuthConfig) (dist.Repository, error) {
  1325. return distribution.GetRepository(ctx, ref, &distribution.ImagePullConfig{
  1326. Config: distribution.Config{
  1327. AuthConfig: authConfig,
  1328. RegistryService: i.registryService,
  1329. },
  1330. })
  1331. }