daemon.go 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895
  1. package main
  2. import (
  3. "context"
  4. "crypto/tls"
  5. "fmt"
  6. "net"
  7. "net/http"
  8. "os"
  9. "path/filepath"
  10. "runtime"
  11. "sort"
  12. "strings"
  13. "sync"
  14. "time"
  15. "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
  16. containerddefaults "github.com/containerd/containerd/defaults"
  17. "github.com/docker/docker/api"
  18. apiserver "github.com/docker/docker/api/server"
  19. buildbackend "github.com/docker/docker/api/server/backend/build"
  20. "github.com/docker/docker/api/server/middleware"
  21. "github.com/docker/docker/api/server/router"
  22. "github.com/docker/docker/api/server/router/build"
  23. checkpointrouter "github.com/docker/docker/api/server/router/checkpoint"
  24. "github.com/docker/docker/api/server/router/container"
  25. distributionrouter "github.com/docker/docker/api/server/router/distribution"
  26. grpcrouter "github.com/docker/docker/api/server/router/grpc"
  27. "github.com/docker/docker/api/server/router/image"
  28. "github.com/docker/docker/api/server/router/network"
  29. pluginrouter "github.com/docker/docker/api/server/router/plugin"
  30. sessionrouter "github.com/docker/docker/api/server/router/session"
  31. swarmrouter "github.com/docker/docker/api/server/router/swarm"
  32. systemrouter "github.com/docker/docker/api/server/router/system"
  33. "github.com/docker/docker/api/server/router/volume"
  34. buildkit "github.com/docker/docker/builder/builder-next"
  35. "github.com/docker/docker/builder/dockerfile"
  36. "github.com/docker/docker/cli/debug"
  37. "github.com/docker/docker/cmd/dockerd/trap"
  38. "github.com/docker/docker/daemon"
  39. "github.com/docker/docker/daemon/cluster"
  40. "github.com/docker/docker/daemon/config"
  41. "github.com/docker/docker/daemon/listeners"
  42. "github.com/docker/docker/dockerversion"
  43. "github.com/docker/docker/libcontainerd/supervisor"
  44. dopts "github.com/docker/docker/opts"
  45. "github.com/docker/docker/pkg/authorization"
  46. "github.com/docker/docker/pkg/homedir"
  47. "github.com/docker/docker/pkg/jsonmessage"
  48. "github.com/docker/docker/pkg/pidfile"
  49. "github.com/docker/docker/pkg/plugingetter"
  50. "github.com/docker/docker/pkg/rootless"
  51. "github.com/docker/docker/pkg/sysinfo"
  52. "github.com/docker/docker/pkg/system"
  53. "github.com/docker/docker/plugin"
  54. "github.com/docker/docker/runconfig"
  55. "github.com/docker/go-connections/tlsconfig"
  56. "github.com/moby/buildkit/session"
  57. swarmapi "github.com/moby/swarmkit/v2/api"
  58. "github.com/pkg/errors"
  59. "github.com/sirupsen/logrus"
  60. "github.com/spf13/pflag"
  61. )
  62. // DaemonCli represents the daemon CLI.
  63. type DaemonCli struct {
  64. *config.Config
  65. configFile *string
  66. flags *pflag.FlagSet
  67. d *daemon.Daemon
  68. authzMiddleware *authorization.Middleware // authzMiddleware enables to dynamically reload the authorization plugins
  69. stopOnce sync.Once
  70. apiShutdown chan struct{}
  71. }
  72. // NewDaemonCli returns a daemon CLI
  73. func NewDaemonCli() *DaemonCli {
  74. return &DaemonCli{
  75. apiShutdown: make(chan struct{}),
  76. }
  77. }
  78. func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
  79. if cli.Config, err = loadDaemonCliConfig(opts); err != nil {
  80. return err
  81. }
  82. tlsConfig, err := newAPIServerTLSConfig(cli.Config)
  83. if err != nil {
  84. return err
  85. }
  86. if opts.Validate {
  87. // If config wasn't OK we wouldn't have made it this far.
  88. _, _ = fmt.Fprintln(os.Stderr, "configuration OK")
  89. return nil
  90. }
  91. configureProxyEnv(cli.Config)
  92. configureDaemonLogs(cli.Config)
  93. logrus.Info("Starting up")
  94. cli.configFile = &opts.configFile
  95. cli.flags = opts.flags
  96. if cli.Config.Debug {
  97. debug.Enable()
  98. }
  99. if cli.Config.Experimental {
  100. logrus.Warn("Running experimental build")
  101. }
  102. if cli.Config.IsRootless() {
  103. logrus.Warn("Running in rootless mode. This mode has feature limitations.")
  104. }
  105. if rootless.RunningWithRootlessKit() {
  106. logrus.Info("Running with RootlessKit integration")
  107. if !cli.Config.IsRootless() {
  108. return fmt.Errorf("rootless mode needs to be enabled for running with RootlessKit")
  109. }
  110. }
  111. // return human-friendly error before creating files
  112. if runtime.GOOS == "linux" && os.Geteuid() != 0 {
  113. return fmt.Errorf("dockerd needs to be started with root privileges. To run dockerd in rootless mode as an unprivileged user, see https://docs.docker.com/go/rootless/")
  114. }
  115. if err := setDefaultUmask(); err != nil {
  116. return err
  117. }
  118. // Create the daemon root before we create ANY other files (PID, or migrate keys)
  119. // to ensure the appropriate ACL is set (particularly relevant on Windows)
  120. if err := daemon.CreateDaemonRoot(cli.Config); err != nil {
  121. return err
  122. }
  123. if err := system.MkdirAll(cli.Config.ExecRoot, 0700); err != nil {
  124. return err
  125. }
  126. potentiallyUnderRuntimeDir := []string{cli.Config.ExecRoot}
  127. if cli.Pidfile != "" {
  128. if err = system.MkdirAll(filepath.Dir(cli.Pidfile), 0o755); err != nil {
  129. return errors.Wrap(err, "failed to create pidfile directory")
  130. }
  131. if err = pidfile.Write(cli.Pidfile, os.Getpid()); err != nil {
  132. return errors.Wrapf(err, "failed to start daemon, ensure docker is not running or delete %s", cli.Pidfile)
  133. }
  134. potentiallyUnderRuntimeDir = append(potentiallyUnderRuntimeDir, cli.Pidfile)
  135. defer func() {
  136. if err := os.Remove(cli.Pidfile); err != nil {
  137. logrus.Error(err)
  138. }
  139. }()
  140. }
  141. if cli.Config.IsRootless() {
  142. // Set sticky bit if XDG_RUNTIME_DIR is set && the file is actually under XDG_RUNTIME_DIR
  143. if _, err := homedir.StickRuntimeDirContents(potentiallyUnderRuntimeDir); err != nil {
  144. // StickRuntimeDirContents returns nil error if XDG_RUNTIME_DIR is just unset
  145. logrus.WithError(err).Warn("cannot set sticky bit on files under XDG_RUNTIME_DIR")
  146. }
  147. }
  148. lss, hosts, err := loadListeners(cli.Config, tlsConfig)
  149. if err != nil {
  150. return errors.Wrap(err, "failed to load listeners")
  151. }
  152. ctx, cancel := context.WithCancel(context.Background())
  153. waitForContainerDShutdown, err := cli.initContainerd(ctx)
  154. if waitForContainerDShutdown != nil {
  155. defer waitForContainerDShutdown(10 * time.Second)
  156. }
  157. if err != nil {
  158. cancel()
  159. return err
  160. }
  161. defer cancel()
  162. httpServer := &http.Server{
  163. ReadHeaderTimeout: 5 * time.Minute, // "G112: Potential Slowloris Attack (gosec)"; not a real concern for our use, so setting a long timeout.
  164. }
  165. apiShutdownCtx, apiShutdownCancel := context.WithCancel(context.Background())
  166. apiShutdownDone := make(chan struct{})
  167. trap.Trap(cli.stop)
  168. go func() {
  169. // Block until cli.stop() has been called.
  170. // It may have already been called, and that's okay.
  171. // Any httpServer.Serve() calls made after
  172. // httpServer.Shutdown() will return immediately,
  173. // which is what we want.
  174. <-cli.apiShutdown
  175. err := httpServer.Shutdown(apiShutdownCtx)
  176. if err != nil {
  177. logrus.WithError(err).Error("Error shutting down http server")
  178. }
  179. close(apiShutdownDone)
  180. }()
  181. defer func() {
  182. select {
  183. case <-cli.apiShutdown:
  184. // cli.stop() has been called and the daemon has completed
  185. // shutting down. Give the HTTP server a little more time to
  186. // finish handling any outstanding requests if needed.
  187. tmr := time.AfterFunc(5*time.Second, apiShutdownCancel)
  188. defer tmr.Stop()
  189. <-apiShutdownDone
  190. default:
  191. // cli.start() has returned without cli.stop() being called,
  192. // e.g. because the daemon failed to start.
  193. // Stop the HTTP server with no grace period.
  194. if closeErr := httpServer.Close(); closeErr != nil {
  195. logrus.WithError(closeErr).Error("Error closing http server")
  196. }
  197. }
  198. }()
  199. // Notify that the API is active, but before daemon is set up.
  200. preNotifyReady()
  201. pluginStore := plugin.NewStore()
  202. var apiServer apiserver.Server
  203. cli.authzMiddleware = initMiddlewares(&apiServer, cli.Config, pluginStore)
  204. d, err := daemon.NewDaemon(ctx, cli.Config, pluginStore, cli.authzMiddleware)
  205. if err != nil {
  206. return errors.Wrap(err, "failed to start daemon")
  207. }
  208. d.StoreHosts(hosts)
  209. // validate after NewDaemon has restored enabled plugins. Don't change order.
  210. if err := validateAuthzPlugins(cli.Config.AuthorizationPlugins, pluginStore); err != nil {
  211. return errors.Wrap(err, "failed to validate authorization plugin")
  212. }
  213. // Note that CDI is not inherrently linux-specific, there are some linux-specific assumptions / implementations in the code that
  214. // queries the properties of device on the host as wel as performs the injection of device nodes and their access permissions into the OCI spec.
  215. //
  216. // In order to lift this restriction the following would have to be addressed:
  217. // - Support needs to be added to the cdi package for injecting Windows devices: https://github.com/container-orchestrated-devices/container-device-interface/issues/28
  218. // - The DeviceRequests API must be extended to non-linux platforms.
  219. if runtime.GOOS == "linux" && cli.Config.Experimental {
  220. daemon.RegisterCDIDriver(
  221. cdi.WithSpecDirs(cli.Config.CDISpecDirs...),
  222. )
  223. }
  224. cli.d = d
  225. if err := startMetricsServer(cli.Config.MetricsAddress); err != nil {
  226. return errors.Wrap(err, "failed to start metrics server")
  227. }
  228. c, err := createAndStartCluster(cli, d)
  229. if err != nil {
  230. logrus.Fatalf("Error starting cluster component: %v", err)
  231. }
  232. // Restart all autostart containers which has a swarm endpoint
  233. // and is not yet running now that we have successfully
  234. // initialized the cluster.
  235. d.RestartSwarmContainers()
  236. logrus.Info("Daemon has completed initialization")
  237. routerCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
  238. defer cancel()
  239. routerOptions, err := newRouterOptions(routerCtx, cli.Config, d)
  240. if err != nil {
  241. return err
  242. }
  243. routerOptions.cluster = c
  244. httpServer.Handler = apiServer.CreateMux(routerOptions.Build()...)
  245. go d.ProcessClusterNotifications(ctx, c.GetWatchStream())
  246. cli.setupConfigReloadTrap()
  247. // after the daemon is done setting up we can notify systemd api
  248. notifyReady()
  249. // Daemon is fully initialized. Start handling API traffic
  250. // and wait for serve API to complete.
  251. var (
  252. apiWG sync.WaitGroup
  253. errAPI = make(chan error, 1)
  254. )
  255. for _, ls := range lss {
  256. apiWG.Add(1)
  257. go func(ls net.Listener) {
  258. defer apiWG.Done()
  259. logrus.Infof("API listen on %s", ls.Addr())
  260. if err := httpServer.Serve(ls); err != http.ErrServerClosed {
  261. logrus.WithFields(logrus.Fields{
  262. logrus.ErrorKey: err,
  263. "listener": ls.Addr(),
  264. }).Error("ServeAPI error")
  265. select {
  266. case errAPI <- err:
  267. default:
  268. }
  269. }
  270. }(ls)
  271. }
  272. apiWG.Wait()
  273. close(errAPI)
  274. c.Cleanup()
  275. // notify systemd that we're shutting down
  276. notifyStopping()
  277. shutdownDaemon(ctx, d)
  278. // Stop notification processing and any background processes
  279. cancel()
  280. if err, ok := <-errAPI; ok {
  281. return errors.Wrap(err, "shutting down due to ServeAPI error")
  282. }
  283. logrus.Info("Daemon shutdown complete")
  284. return nil
  285. }
  286. type routerOptions struct {
  287. sessionManager *session.Manager
  288. buildBackend *buildbackend.Backend
  289. features func() map[string]bool
  290. buildkit *buildkit.Builder
  291. daemon *daemon.Daemon
  292. cluster *cluster.Cluster
  293. }
  294. func newRouterOptions(ctx context.Context, config *config.Config, d *daemon.Daemon) (routerOptions, error) {
  295. opts := routerOptions{}
  296. sm, err := session.NewManager()
  297. if err != nil {
  298. return opts, errors.Wrap(err, "failed to create sessionmanager")
  299. }
  300. manager, err := dockerfile.NewBuildManager(d.BuilderBackend(), d.IdentityMapping())
  301. if err != nil {
  302. return opts, err
  303. }
  304. cgroupParent := newCgroupParent(config)
  305. ro := routerOptions{
  306. sessionManager: sm,
  307. features: d.Features,
  308. daemon: d,
  309. }
  310. bk, err := buildkit.New(ctx, buildkit.Opt{
  311. SessionManager: sm,
  312. Root: filepath.Join(config.Root, "buildkit"),
  313. EngineID: d.ID(),
  314. Dist: d.DistributionServices(),
  315. ImageTagger: d.ImageService(),
  316. NetworkController: d.NetworkController(),
  317. DefaultCgroupParent: cgroupParent,
  318. RegistryHosts: d.RegistryHosts,
  319. BuilderConfig: config.Builder,
  320. Rootless: daemon.Rootless(config),
  321. IdentityMapping: d.IdentityMapping(),
  322. DNSConfig: config.DNSConfig,
  323. ApparmorProfile: daemon.DefaultApparmorProfile(),
  324. UseSnapshotter: d.UsesSnapshotter(),
  325. Snapshotter: d.ImageService().StorageDriver(),
  326. ContainerdAddress: config.ContainerdAddr,
  327. ContainerdNamespace: config.ContainerdNamespace,
  328. })
  329. if err != nil {
  330. return opts, err
  331. }
  332. bb, err := buildbackend.NewBackend(d.ImageService(), manager, bk, d.EventsService)
  333. if err != nil {
  334. return opts, errors.Wrap(err, "failed to create buildmanager")
  335. }
  336. ro.buildBackend = bb
  337. ro.buildkit = bk
  338. return ro, nil
  339. }
  340. func (cli *DaemonCli) reloadConfig() {
  341. reload := func(c *config.Config) {
  342. if err := validateAuthzPlugins(c.AuthorizationPlugins, cli.d.PluginStore); err != nil {
  343. logrus.Fatalf("Error validating authorization plugin: %v", err)
  344. return
  345. }
  346. if err := cli.d.Reload(c); err != nil {
  347. logrus.Errorf("Error reconfiguring the daemon: %v", err)
  348. return
  349. }
  350. // Apply our own configuration only after the daemon reload has succeeded. We
  351. // don't want to partially apply the config if the daemon is unhappy with it.
  352. cli.authzMiddleware.SetPlugins(c.AuthorizationPlugins)
  353. if c.IsValueSet("debug") {
  354. debugEnabled := debug.IsEnabled()
  355. switch {
  356. case debugEnabled && !c.Debug: // disable debug
  357. debug.Disable()
  358. case c.Debug && !debugEnabled: // enable debug
  359. debug.Enable()
  360. }
  361. }
  362. }
  363. if err := config.Reload(*cli.configFile, cli.flags, reload); err != nil {
  364. logrus.Error(err)
  365. }
  366. }
  367. func (cli *DaemonCli) stop() {
  368. // Signal that the API server should shut down as soon as possible.
  369. // This construct is used rather than directly shutting down the HTTP
  370. // server to avoid any issues if this method is called before the server
  371. // has been instantiated in cli.start(). If this method is called first,
  372. // the HTTP server will be shut down immediately upon instantiation.
  373. cli.stopOnce.Do(func() {
  374. close(cli.apiShutdown)
  375. })
  376. }
  377. // shutdownDaemon just wraps daemon.Shutdown() to handle a timeout in case
  378. // d.Shutdown() is waiting too long to kill container or worst it's
  379. // blocked there
  380. func shutdownDaemon(ctx context.Context, d *daemon.Daemon) {
  381. var cancel context.CancelFunc
  382. if timeout := d.ShutdownTimeout(); timeout >= 0 {
  383. ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout)*time.Second)
  384. } else {
  385. ctx, cancel = context.WithCancel(ctx)
  386. }
  387. go func() {
  388. defer cancel()
  389. d.Shutdown(ctx)
  390. }()
  391. <-ctx.Done()
  392. if errors.Is(ctx.Err(), context.DeadlineExceeded) {
  393. logrus.Error("Force shutdown daemon")
  394. } else {
  395. logrus.Debug("Clean shutdown succeeded")
  396. }
  397. }
  398. func loadDaemonCliConfig(opts *daemonOptions) (*config.Config, error) {
  399. if !opts.flags.Parsed() {
  400. return nil, errors.New(`cannot load CLI config before flags are parsed`)
  401. }
  402. opts.setDefaultOptions()
  403. conf := opts.daemonConfig
  404. flags := opts.flags
  405. conf.Debug = opts.Debug
  406. conf.Hosts = opts.Hosts
  407. conf.LogLevel = opts.LogLevel
  408. if flags.Changed(FlagTLS) {
  409. conf.TLS = &opts.TLS
  410. }
  411. if flags.Changed(FlagTLSVerify) {
  412. conf.TLSVerify = &opts.TLSVerify
  413. v := true
  414. conf.TLS = &v
  415. }
  416. if opts.TLSOptions != nil {
  417. conf.TLSOptions = config.TLSOptions{
  418. CAFile: opts.TLSOptions.CAFile,
  419. CertFile: opts.TLSOptions.CertFile,
  420. KeyFile: opts.TLSOptions.KeyFile,
  421. }
  422. } else {
  423. conf.TLSOptions = config.TLSOptions{}
  424. }
  425. if opts.configFile != "" {
  426. c, err := config.MergeDaemonConfigurations(conf, flags, opts.configFile)
  427. if err != nil {
  428. if flags.Changed("config-file") || !os.IsNotExist(err) {
  429. return nil, errors.Wrapf(err, "unable to configure the Docker daemon with file %s", opts.configFile)
  430. }
  431. }
  432. // the merged configuration can be nil if the config file didn't exist.
  433. // leave the current configuration as it is if when that happens.
  434. if c != nil {
  435. conf = c
  436. }
  437. }
  438. if err := normalizeHosts(conf); err != nil {
  439. return nil, err
  440. }
  441. if err := config.Validate(conf); err != nil {
  442. return nil, err
  443. }
  444. // Check if duplicate label-keys with different values are found
  445. newLabels, err := config.GetConflictFreeLabels(conf.Labels)
  446. if err != nil {
  447. return nil, err
  448. }
  449. conf.Labels = newLabels
  450. // Regardless of whether the user sets it to true or false, if they
  451. // specify TLSVerify at all then we need to turn on TLS
  452. if conf.IsValueSet(FlagTLSVerify) {
  453. v := true
  454. conf.TLS = &v
  455. }
  456. if conf.TLSVerify == nil && conf.TLS != nil {
  457. conf.TLSVerify = conf.TLS
  458. }
  459. err = validateCPURealtimeOptions(conf)
  460. if err != nil {
  461. return nil, err
  462. }
  463. return conf, nil
  464. }
  465. // normalizeHosts normalizes the configured config.Hosts and remove duplicates.
  466. // It returns an error if it fails to parse a host.
  467. func normalizeHosts(config *config.Config) error {
  468. if len(config.Hosts) == 0 {
  469. // if no hosts are configured, create a single entry slice, so that the
  470. // default is used.
  471. //
  472. // TODO(thaJeztah) implement a cleaner way for this; this depends on a
  473. // side-effect of how we parse empty/partial hosts.
  474. config.Hosts = make([]string, 1)
  475. }
  476. hosts := make([]string, 0, len(config.Hosts))
  477. seen := make(map[string]struct{}, len(config.Hosts))
  478. useTLS := DefaultTLSValue
  479. if config.TLS != nil {
  480. useTLS = *config.TLS
  481. }
  482. for _, h := range config.Hosts {
  483. host, err := dopts.ParseHost(useTLS, honorXDG, h)
  484. if err != nil {
  485. return err
  486. }
  487. if _, ok := seen[host]; ok {
  488. continue
  489. }
  490. seen[host] = struct{}{}
  491. hosts = append(hosts, host)
  492. }
  493. sort.Strings(hosts)
  494. config.Hosts = hosts
  495. return nil
  496. }
  497. func (opts routerOptions) Build() []router.Router {
  498. decoder := runconfig.ContainerDecoder{
  499. GetSysInfo: func() *sysinfo.SysInfo {
  500. return opts.daemon.RawSysInfo()
  501. },
  502. }
  503. routers := []router.Router{
  504. // we need to add the checkpoint router before the container router or the DELETE gets masked
  505. checkpointrouter.NewRouter(opts.daemon, decoder),
  506. container.NewRouter(opts.daemon, decoder, opts.daemon.RawSysInfo().CgroupUnified),
  507. image.NewRouter(
  508. opts.daemon.ImageService(),
  509. opts.daemon.RegistryService(),
  510. opts.daemon.ReferenceStore,
  511. opts.daemon.ImageService().DistributionServices().ImageStore,
  512. opts.daemon.ImageService().DistributionServices().LayerStore,
  513. ),
  514. systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildkit, opts.daemon.Features),
  515. volume.NewRouter(opts.daemon.VolumesService(), opts.cluster),
  516. build.NewRouter(opts.buildBackend, opts.daemon),
  517. sessionrouter.NewRouter(opts.sessionManager),
  518. swarmrouter.NewRouter(opts.cluster),
  519. pluginrouter.NewRouter(opts.daemon.PluginManager()),
  520. distributionrouter.NewRouter(opts.daemon.ImageBackend()),
  521. }
  522. if opts.buildBackend != nil {
  523. routers = append(routers, grpcrouter.NewRouter(opts.buildBackend))
  524. }
  525. if opts.daemon.NetworkControllerEnabled() {
  526. routers = append(routers, network.NewRouter(opts.daemon, opts.cluster))
  527. }
  528. if opts.daemon.HasExperimental() {
  529. for _, r := range routers {
  530. for _, route := range r.Routes() {
  531. if experimental, ok := route.(router.ExperimentalRoute); ok {
  532. experimental.Enable()
  533. }
  534. }
  535. }
  536. }
  537. return routers
  538. }
  539. func initMiddlewares(s *apiserver.Server, cfg *config.Config, pluginStore plugingetter.PluginGetter) *authorization.Middleware {
  540. v := dockerversion.Version
  541. exp := middleware.NewExperimentalMiddleware(cfg.Experimental)
  542. s.UseMiddleware(exp)
  543. vm := middleware.NewVersionMiddleware(v, api.DefaultVersion, api.MinVersion)
  544. s.UseMiddleware(vm)
  545. if cfg.CorsHeaders != "" {
  546. c := middleware.NewCORSMiddleware(cfg.CorsHeaders)
  547. s.UseMiddleware(c)
  548. }
  549. authzMiddleware := authorization.NewMiddleware(cfg.AuthorizationPlugins, pluginStore)
  550. s.UseMiddleware(authzMiddleware)
  551. return authzMiddleware
  552. }
  553. func (cli *DaemonCli) getContainerdDaemonOpts() ([]supervisor.DaemonOpt, error) {
  554. var opts []supervisor.DaemonOpt
  555. if cli.Debug {
  556. opts = append(opts, supervisor.WithLogLevel("debug"))
  557. } else {
  558. opts = append(opts, supervisor.WithLogLevel(cli.LogLevel))
  559. }
  560. if !cli.CriContainerd {
  561. // CRI support in the managed daemon is currently opt-in.
  562. //
  563. // It's disabled by default, originally because it was listening on
  564. // a TCP connection at 0.0.0.0:10010, which was considered a security
  565. // risk, and could conflict with user's container ports.
  566. //
  567. // Current versions of containerd started now listen on localhost on
  568. // an ephemeral port instead, but could still conflict with container
  569. // ports, and running kubernetes using the static binaries is not a
  570. // common scenario, so we (for now) continue disabling it by default.
  571. //
  572. // Also see https://github.com/containerd/containerd/issues/2483#issuecomment-407530608
  573. opts = append(opts, supervisor.WithCRIDisabled())
  574. }
  575. return opts, nil
  576. }
  577. func newAPIServerTLSConfig(config *config.Config) (*tls.Config, error) {
  578. var tlsConfig *tls.Config
  579. if config.TLS != nil && *config.TLS {
  580. var (
  581. clientAuth tls.ClientAuthType
  582. err error
  583. )
  584. if config.TLSVerify == nil || *config.TLSVerify {
  585. // server requires and verifies client's certificate
  586. clientAuth = tls.RequireAndVerifyClientCert
  587. }
  588. tlsConfig, err = tlsconfig.Server(tlsconfig.Options{
  589. CAFile: config.TLSOptions.CAFile,
  590. CertFile: config.TLSOptions.CertFile,
  591. KeyFile: config.TLSOptions.KeyFile,
  592. ExclusiveRootPools: true,
  593. ClientAuth: clientAuth,
  594. })
  595. if err != nil {
  596. return nil, errors.Wrap(err, "invalid TLS configuration")
  597. }
  598. }
  599. return tlsConfig, nil
  600. }
  601. // checkTLSAuthOK checks basically for an explicitly disabled TLS/TLSVerify
  602. // Going forward we do not want to support a scenario where dockerd listens
  603. // on TCP without either TLS client auth (or an explicit opt-in to disable it)
  604. func checkTLSAuthOK(c *config.Config) bool {
  605. if c.TLS == nil {
  606. // Either TLS is enabled by default, in which case TLS verification should be enabled by default, or explicitly disabled
  607. // Or TLS is disabled by default... in any of these cases, we can just take the default value as to how to proceed
  608. return DefaultTLSValue
  609. }
  610. if !*c.TLS {
  611. // TLS is explicitly disabled, which is supported
  612. return true
  613. }
  614. if c.TLSVerify == nil {
  615. // this actually shouldn't happen since we set TLSVerify on the config object anyway
  616. // But in case it does get here, be cautious and assume this is not supported.
  617. return false
  618. }
  619. // Either TLSVerify is explicitly enabled or disabled, both cases are supported
  620. return true
  621. }
  622. func loadListeners(cfg *config.Config, tlsConfig *tls.Config) ([]net.Listener, []string, error) {
  623. if len(cfg.Hosts) == 0 {
  624. return nil, nil, errors.New("no hosts configured")
  625. }
  626. var (
  627. hosts []string
  628. lss []net.Listener
  629. )
  630. for i := 0; i < len(cfg.Hosts); i++ {
  631. protoAddr := cfg.Hosts[i]
  632. proto, addr, ok := strings.Cut(protoAddr, "://")
  633. if !ok {
  634. return nil, nil, fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
  635. }
  636. // It's a bad idea to bind to TCP without tlsverify.
  637. authEnabled := tlsConfig != nil && tlsConfig.ClientAuth == tls.RequireAndVerifyClientCert
  638. if proto == "tcp" && !authEnabled {
  639. logrus.WithField("host", protoAddr).Warn("Binding to IP address without --tlsverify is insecure and gives root access on this machine to everyone who has access to your network.")
  640. logrus.WithField("host", protoAddr).Warn("Binding to an IP address, even on localhost, can also give access to scripts run in a browser. Be safe out there!")
  641. time.Sleep(time.Second)
  642. // If TLSVerify is explicitly set to false we'll take that as "Please let me shoot myself in the foot"
  643. // We do not want to continue to support a default mode where tls verification is disabled, so we do some extra warnings here and eventually remove support
  644. if !checkTLSAuthOK(cfg) {
  645. ipAddr, _, err := net.SplitHostPort(addr)
  646. if err != nil {
  647. return nil, nil, errors.Wrap(err, "error parsing tcp address")
  648. }
  649. // shortcut all this extra stuff for literal "localhost"
  650. // -H supports specifying hostnames, since we want to bypass this on loopback interfaces we'll look it up here.
  651. if ipAddr != "localhost" {
  652. ip := net.ParseIP(ipAddr)
  653. if ip == nil {
  654. ipA, err := net.ResolveIPAddr("ip", ipAddr)
  655. if err != nil {
  656. logrus.WithError(err).WithField("host", ipAddr).Error("Error looking up specified host address")
  657. }
  658. if ipA != nil {
  659. ip = ipA.IP
  660. }
  661. }
  662. if ip == nil || !ip.IsLoopback() {
  663. logrus.WithField("host", protoAddr).Warn("Binding to an IP address without --tlsverify is deprecated. Startup is intentionally being slowed down to show this message")
  664. logrus.WithField("host", protoAddr).Warn("Please consider generating tls certificates with client validation to prevent exposing unauthenticated root access to your network")
  665. logrus.WithField("host", protoAddr).Warnf("You can override this by explicitly specifying '--%s=false' or '--%s=false'", FlagTLS, FlagTLSVerify)
  666. logrus.WithField("host", protoAddr).Warnf("Support for listening on TCP without authentication or explicit intent to run without authentication will be removed in the next release")
  667. time.Sleep(15 * time.Second)
  668. }
  669. }
  670. }
  671. }
  672. // If we're binding to a TCP port, make sure that a container doesn't try to use it.
  673. if proto == "tcp" {
  674. if err := allocateDaemonPort(addr); err != nil {
  675. return nil, nil, err
  676. }
  677. }
  678. ls, err := listeners.Init(proto, addr, cfg.SocketGroup, tlsConfig)
  679. if err != nil {
  680. return nil, nil, err
  681. }
  682. logrus.Debugf("Listener created for HTTP on %s (%s)", proto, addr)
  683. hosts = append(hosts, addr)
  684. lss = append(lss, ls...)
  685. }
  686. return lss, hosts, nil
  687. }
  688. func createAndStartCluster(cli *DaemonCli, d *daemon.Daemon) (*cluster.Cluster, error) {
  689. name, _ := os.Hostname()
  690. // Use a buffered channel to pass changes from store watch API to daemon
  691. // A buffer allows store watch API and daemon processing to not wait for each other
  692. watchStream := make(chan *swarmapi.WatchMessage, 32)
  693. c, err := cluster.New(cluster.Config{
  694. Root: cli.Config.Root,
  695. Name: name,
  696. Backend: d,
  697. VolumeBackend: d.VolumesService(),
  698. ImageBackend: d.ImageBackend(),
  699. PluginBackend: d.PluginManager(),
  700. NetworkSubnetsProvider: d,
  701. DefaultAdvertiseAddr: cli.Config.SwarmDefaultAdvertiseAddr,
  702. RaftHeartbeatTick: cli.Config.SwarmRaftHeartbeatTick,
  703. RaftElectionTick: cli.Config.SwarmRaftElectionTick,
  704. RuntimeRoot: cli.getSwarmRunRoot(),
  705. WatchStream: watchStream,
  706. })
  707. if err != nil {
  708. return nil, err
  709. }
  710. d.SetCluster(c)
  711. err = c.Start()
  712. return c, err
  713. }
  714. // validates that the plugins requested with the --authorization-plugin flag are valid AuthzDriver
  715. // plugins present on the host and available to the daemon
  716. func validateAuthzPlugins(requestedPlugins []string, pg plugingetter.PluginGetter) error {
  717. for _, reqPlugin := range requestedPlugins {
  718. if _, err := pg.Get(reqPlugin, authorization.AuthZApiImplements, plugingetter.Lookup); err != nil {
  719. return err
  720. }
  721. }
  722. return nil
  723. }
  724. func systemContainerdRunning(honorXDG bool) (string, bool, error) {
  725. addr := containerddefaults.DefaultAddress
  726. if honorXDG {
  727. runtimeDir, err := homedir.GetRuntimeDir()
  728. if err != nil {
  729. return "", false, err
  730. }
  731. addr = filepath.Join(runtimeDir, "containerd", "containerd.sock")
  732. }
  733. _, err := os.Lstat(addr)
  734. return addr, err == nil, nil
  735. }
  736. // configureDaemonLogs sets the logrus logging level and formatting. It expects
  737. // the passed configuration to already be validated, and ignores invalid options.
  738. func configureDaemonLogs(conf *config.Config) {
  739. if conf.LogLevel != "" {
  740. lvl, err := logrus.ParseLevel(conf.LogLevel)
  741. if err == nil {
  742. logrus.SetLevel(lvl)
  743. }
  744. } else {
  745. logrus.SetLevel(logrus.InfoLevel)
  746. }
  747. logrus.SetFormatter(&logrus.TextFormatter{
  748. TimestampFormat: jsonmessage.RFC3339NanoFixed,
  749. DisableColors: conf.RawLogs,
  750. FullTimestamp: true,
  751. })
  752. }
  753. func configureProxyEnv(conf *config.Config) {
  754. if p := conf.HTTPProxy; p != "" {
  755. overrideProxyEnv("HTTP_PROXY", p)
  756. overrideProxyEnv("http_proxy", p)
  757. }
  758. if p := conf.HTTPSProxy; p != "" {
  759. overrideProxyEnv("HTTPS_PROXY", p)
  760. overrideProxyEnv("https_proxy", p)
  761. }
  762. if p := conf.NoProxy; p != "" {
  763. overrideProxyEnv("NO_PROXY", p)
  764. overrideProxyEnv("no_proxy", p)
  765. }
  766. }
  767. func overrideProxyEnv(name, val string) {
  768. if oldVal := os.Getenv(name); oldVal != "" && oldVal != val {
  769. logrus.WithFields(logrus.Fields{
  770. "name": name,
  771. "old-value": config.MaskCredentials(oldVal),
  772. "new-value": config.MaskCredentials(val),
  773. }).Warn("overriding existing proxy variable with value from configuration")
  774. }
  775. _ = os.Setenv(name, val)
  776. }