controller.go 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312
  1. /*
  2. Package libnetwork provides the basic functionality and extension points to
  3. create network namespaces and allocate interfaces for containers to use.
  4. networkType := "bridge"
  5. // Create a new controller instance
  6. driverOptions := options.Generic{}
  7. genericOption := make(map[string]interface{})
  8. genericOption[netlabel.GenericData] = driverOptions
  9. controller, err := libnetwork.New(config.OptionDriverConfig(networkType, genericOption))
  10. if err != nil {
  11. return
  12. }
  13. // Create a network for containers to join.
  14. // NewNetwork accepts Variadic optional arguments that libnetwork and Drivers can make use of
  15. network, err := controller.NewNetwork(networkType, "network1", "")
  16. if err != nil {
  17. return
  18. }
  19. // For each new container: allocate IP and interfaces. The returned network
  20. // settings will be used for container infos (inspect and such), as well as
  21. // iptables rules for port publishing. This info is contained or accessible
  22. // from the returned endpoint.
  23. ep, err := network.CreateEndpoint("Endpoint1")
  24. if err != nil {
  25. return
  26. }
  27. // Create the sandbox for the container.
  28. // NewSandbox accepts Variadic optional arguments which libnetwork can use.
  29. sbx, err := controller.NewSandbox("container1",
  30. libnetwork.OptionHostname("test"),
  31. libnetwork.OptionDomainname("docker.io"))
  32. // A sandbox can join the endpoint via the join api.
  33. err = ep.Join(sbx)
  34. if err != nil {
  35. return
  36. }
  37. */
  38. package libnetwork
  39. import (
  40. "fmt"
  41. "net"
  42. "path/filepath"
  43. "runtime"
  44. "strings"
  45. "sync"
  46. "time"
  47. "github.com/docker/docker/libnetwork/cluster"
  48. "github.com/docker/docker/libnetwork/config"
  49. "github.com/docker/docker/libnetwork/datastore"
  50. "github.com/docker/docker/libnetwork/diagnostic"
  51. "github.com/docker/docker/libnetwork/discoverapi"
  52. "github.com/docker/docker/libnetwork/driverapi"
  53. "github.com/docker/docker/libnetwork/drvregistry"
  54. "github.com/docker/docker/libnetwork/ipamapi"
  55. "github.com/docker/docker/libnetwork/netlabel"
  56. "github.com/docker/docker/libnetwork/options"
  57. "github.com/docker/docker/libnetwork/osl"
  58. "github.com/docker/docker/libnetwork/types"
  59. "github.com/docker/docker/pkg/plugingetter"
  60. "github.com/docker/docker/pkg/plugins"
  61. "github.com/docker/docker/pkg/stringid"
  62. "github.com/moby/locker"
  63. "github.com/pkg/errors"
  64. "github.com/sirupsen/logrus"
  65. )
  66. // NetworkController provides the interface for controller instance which manages
  67. // networks.
  68. type NetworkController interface {
  69. // ID provides a unique identity for the controller
  70. ID() string
  71. // BuiltinDrivers returns list of builtin drivers
  72. BuiltinDrivers() []string
  73. // BuiltinIPAMDrivers returns list of builtin ipam drivers
  74. BuiltinIPAMDrivers() []string
  75. // Config method returns the bootup configuration for the controller
  76. Config() config.Config
  77. // Create a new network. The options parameter carries network specific options.
  78. NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error)
  79. // Networks returns the list of Network(s) managed by this controller.
  80. Networks() []Network
  81. // WalkNetworks uses the provided function to walk the Network(s) managed by this controller.
  82. WalkNetworks(walker NetworkWalker)
  83. // NetworkByName returns the Network which has the passed name. If not found, the error ErrNoSuchNetwork is returned.
  84. NetworkByName(name string) (Network, error)
  85. // NetworkByID returns the Network which has the passed id. If not found, the error ErrNoSuchNetwork is returned.
  86. NetworkByID(id string) (Network, error)
  87. // NewSandbox creates a new network sandbox for the passed container id
  88. NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error)
  89. // Sandboxes returns the list of Sandbox(s) managed by this controller.
  90. Sandboxes() []Sandbox
  91. // WalkSandboxes uses the provided function to walk the Sandbox(s) managed by this controller.
  92. WalkSandboxes(walker SandboxWalker)
  93. // SandboxByID returns the Sandbox which has the passed id. If not found, a types.NotFoundError is returned.
  94. SandboxByID(id string) (Sandbox, error)
  95. // SandboxDestroy destroys a sandbox given a container ID
  96. SandboxDestroy(id string) error
  97. // Stop network controller
  98. Stop()
  99. // ReloadConfiguration updates the controller configuration
  100. ReloadConfiguration(cfgOptions ...config.Option) error
  101. // SetClusterProvider sets cluster provider
  102. SetClusterProvider(provider cluster.Provider)
  103. // Wait for agent initialization complete in libnetwork controller
  104. AgentInitWait()
  105. // Wait for agent to stop if running
  106. AgentStopWait()
  107. // SetKeys configures the encryption key for gossip and overlay data path
  108. SetKeys(keys []*types.EncryptionKey) error
  109. // StartDiagnostic start the network diagnostic mode
  110. StartDiagnostic(port int)
  111. // StopDiagnostic start the network diagnostic mode
  112. StopDiagnostic()
  113. // IsDiagnosticEnabled returns true if the diagnostic is enabled
  114. IsDiagnosticEnabled() bool
  115. }
  116. // NetworkWalker is a client provided function which will be used to walk the Networks.
  117. // When the function returns true, the walk will stop.
  118. type NetworkWalker func(nw Network) bool
  119. // SandboxWalker is a client provided function which will be used to walk the Sandboxes.
  120. // When the function returns true, the walk will stop.
  121. type SandboxWalker func(sb Sandbox) bool
  122. type sandboxTable map[string]*sandbox
  123. type controller struct {
  124. id string
  125. drvRegistry *drvregistry.DrvRegistry
  126. sandboxes sandboxTable
  127. cfg *config.Config
  128. stores []datastore.DataStore
  129. extKeyListener net.Listener
  130. watchCh chan *endpoint
  131. unWatchCh chan *endpoint
  132. svcRecords map[string]svcInfo
  133. nmap map[string]*netWatch
  134. serviceBindings map[serviceKey]*service
  135. defOsSbox osl.Sandbox
  136. ingressSandbox *sandbox
  137. sboxOnce sync.Once
  138. agent *agent
  139. networkLocker *locker.Locker
  140. agentInitDone chan struct{}
  141. agentStopDone chan struct{}
  142. keys []*types.EncryptionKey
  143. DiagnosticServer *diagnostic.Server
  144. sync.Mutex
  145. }
  146. type initializer struct {
  147. fn drvregistry.InitFunc
  148. ntype string
  149. }
  150. // New creates a new instance of network controller.
  151. func New(cfgOptions ...config.Option) (NetworkController, error) {
  152. c := &controller{
  153. id: stringid.GenerateRandomID(),
  154. cfg: config.ParseConfigOptions(cfgOptions...),
  155. sandboxes: sandboxTable{},
  156. svcRecords: make(map[string]svcInfo),
  157. serviceBindings: make(map[serviceKey]*service),
  158. agentInitDone: make(chan struct{}),
  159. networkLocker: locker.New(),
  160. DiagnosticServer: diagnostic.New(),
  161. }
  162. c.DiagnosticServer.Init()
  163. if err := c.initStores(); err != nil {
  164. return nil, err
  165. }
  166. drvRegistry, err := drvregistry.New(c.getStore(datastore.LocalScope), c.getStore(datastore.GlobalScope), c.RegisterDriver, nil, c.cfg.PluginGetter)
  167. if err != nil {
  168. return nil, err
  169. }
  170. for _, i := range getInitializers(c.cfg.Daemon.Experimental) {
  171. var dcfg map[string]interface{}
  172. // External plugins don't need config passed through daemon. They can
  173. // bootstrap themselves
  174. if i.ntype != "remote" {
  175. dcfg = c.makeDriverConfig(i.ntype)
  176. }
  177. if err := drvRegistry.AddDriver(i.ntype, i.fn, dcfg); err != nil {
  178. return nil, err
  179. }
  180. }
  181. if err = initIPAMDrivers(drvRegistry, nil, c.getStore(datastore.GlobalScope), c.cfg.Daemon.DefaultAddressPool); err != nil {
  182. return nil, err
  183. }
  184. c.drvRegistry = drvRegistry
  185. c.WalkNetworks(populateSpecial)
  186. // Reserve pools first before doing cleanup. Otherwise the
  187. // cleanups of endpoint/network and sandbox below will
  188. // generate many unnecessary warnings
  189. c.reservePools()
  190. // Cleanup resources
  191. c.sandboxCleanup(c.cfg.ActiveSandboxes)
  192. c.cleanupLocalEndpoints()
  193. c.networkCleanup()
  194. if err := c.startExternalKeyListener(); err != nil {
  195. return nil, err
  196. }
  197. setupArrangeUserFilterRule(c)
  198. return c, nil
  199. }
  200. func (c *controller) SetClusterProvider(provider cluster.Provider) {
  201. var sameProvider bool
  202. c.Lock()
  203. // Avoids to spawn multiple goroutine for the same cluster provider
  204. if c.cfg.Daemon.ClusterProvider == provider {
  205. // If the cluster provider is already set, there is already a go routine spawned
  206. // that is listening for events, so nothing to do here
  207. sameProvider = true
  208. } else {
  209. c.cfg.Daemon.ClusterProvider = provider
  210. }
  211. c.Unlock()
  212. if provider == nil || sameProvider {
  213. return
  214. }
  215. // We don't want to spawn a new go routine if the previous one did not exit yet
  216. c.AgentStopWait()
  217. go c.clusterAgentInit()
  218. }
  219. // libnetwork side of agent depends on the keys. On the first receipt of
  220. // keys setup the agent. For subsequent key set handle the key change
  221. func (c *controller) SetKeys(keys []*types.EncryptionKey) error {
  222. subsysKeys := make(map[string]int)
  223. for _, key := range keys {
  224. if key.Subsystem != subsysGossip &&
  225. key.Subsystem != subsysIPSec {
  226. return fmt.Errorf("key received for unrecognized subsystem")
  227. }
  228. subsysKeys[key.Subsystem]++
  229. }
  230. for s, count := range subsysKeys {
  231. if count != keyringSize {
  232. return fmt.Errorf("incorrect number of keys for subsystem %v", s)
  233. }
  234. }
  235. agent := c.getAgent()
  236. if agent == nil {
  237. c.Lock()
  238. c.keys = keys
  239. c.Unlock()
  240. return nil
  241. }
  242. return c.handleKeyChange(keys)
  243. }
  244. func (c *controller) getAgent() *agent {
  245. c.Lock()
  246. defer c.Unlock()
  247. return c.agent
  248. }
  249. func (c *controller) clusterAgentInit() {
  250. clusterProvider := c.cfg.Daemon.ClusterProvider
  251. var keysAvailable bool
  252. for {
  253. eventType := <-clusterProvider.ListenClusterEvents()
  254. // The events: EventSocketChange, EventNodeReady and EventNetworkKeysAvailable are not ordered
  255. // when all the condition for the agent initialization are met then proceed with it
  256. switch eventType {
  257. case cluster.EventNetworkKeysAvailable:
  258. // Validates that the keys are actually available before starting the initialization
  259. // This will handle old spurious messages left on the channel
  260. c.Lock()
  261. keysAvailable = c.keys != nil
  262. c.Unlock()
  263. fallthrough
  264. case cluster.EventSocketChange, cluster.EventNodeReady:
  265. if keysAvailable && !c.isDistributedControl() {
  266. c.agentOperationStart()
  267. if err := c.agentSetup(clusterProvider); err != nil {
  268. c.agentStopComplete()
  269. } else {
  270. c.agentInitComplete()
  271. }
  272. }
  273. case cluster.EventNodeLeave:
  274. c.agentOperationStart()
  275. c.Lock()
  276. c.keys = nil
  277. c.Unlock()
  278. // We are leaving the cluster. Make sure we
  279. // close the gossip so that we stop all
  280. // incoming gossip updates before cleaning up
  281. // any remaining service bindings. But before
  282. // deleting the networks since the networks
  283. // should still be present when cleaning up
  284. // service bindings
  285. c.agentClose()
  286. c.cleanupServiceDiscovery("")
  287. c.cleanupServiceBindings("")
  288. c.agentStopComplete()
  289. return
  290. }
  291. }
  292. }
  293. // AgentInitWait waits for agent initialization to be completed in the controller.
  294. func (c *controller) AgentInitWait() {
  295. c.Lock()
  296. agentInitDone := c.agentInitDone
  297. c.Unlock()
  298. if agentInitDone != nil {
  299. <-agentInitDone
  300. }
  301. }
  302. // AgentStopWait waits for the Agent stop to be completed in the controller
  303. func (c *controller) AgentStopWait() {
  304. c.Lock()
  305. agentStopDone := c.agentStopDone
  306. c.Unlock()
  307. if agentStopDone != nil {
  308. <-agentStopDone
  309. }
  310. }
  311. // agentOperationStart marks the start of an Agent Init or Agent Stop
  312. func (c *controller) agentOperationStart() {
  313. c.Lock()
  314. if c.agentInitDone == nil {
  315. c.agentInitDone = make(chan struct{})
  316. }
  317. if c.agentStopDone == nil {
  318. c.agentStopDone = make(chan struct{})
  319. }
  320. c.Unlock()
  321. }
  322. // agentInitComplete notifies the successful completion of the Agent initialization
  323. func (c *controller) agentInitComplete() {
  324. c.Lock()
  325. if c.agentInitDone != nil {
  326. close(c.agentInitDone)
  327. c.agentInitDone = nil
  328. }
  329. c.Unlock()
  330. }
  331. // agentStopComplete notifies the successful completion of the Agent stop
  332. func (c *controller) agentStopComplete() {
  333. c.Lock()
  334. if c.agentStopDone != nil {
  335. close(c.agentStopDone)
  336. c.agentStopDone = nil
  337. }
  338. c.Unlock()
  339. }
  340. func (c *controller) makeDriverConfig(ntype string) map[string]interface{} {
  341. if c.cfg == nil {
  342. return nil
  343. }
  344. config := make(map[string]interface{})
  345. for _, label := range c.cfg.Daemon.Labels {
  346. if !strings.HasPrefix(netlabel.Key(label), netlabel.DriverPrefix+"."+ntype) {
  347. continue
  348. }
  349. config[netlabel.Key(label)] = netlabel.Value(label)
  350. }
  351. drvCfg, ok := c.cfg.Daemon.DriverCfg[ntype]
  352. if ok {
  353. for k, v := range drvCfg.(map[string]interface{}) {
  354. config[k] = v
  355. }
  356. }
  357. for k, v := range c.cfg.Scopes {
  358. if !v.IsValid() {
  359. continue
  360. }
  361. config[netlabel.MakeKVClient(k)] = discoverapi.DatastoreConfigData{
  362. Scope: k,
  363. Provider: v.Client.Provider,
  364. Address: v.Client.Address,
  365. Config: v.Client.Config,
  366. }
  367. }
  368. return config
  369. }
  370. var procReloadConfig = make(chan (bool), 1)
  371. func (c *controller) ReloadConfiguration(cfgOptions ...config.Option) error {
  372. procReloadConfig <- true
  373. defer func() { <-procReloadConfig }()
  374. // For now we accept the configuration reload only as a mean to provide a global store config after boot.
  375. // Refuse the configuration if it alters an existing datastore client configuration.
  376. update := false
  377. cfg := config.ParseConfigOptions(cfgOptions...)
  378. for s := range c.cfg.Scopes {
  379. if _, ok := cfg.Scopes[s]; !ok {
  380. return types.ForbiddenErrorf("cannot accept new configuration because it removes an existing datastore client")
  381. }
  382. }
  383. for s, nSCfg := range cfg.Scopes {
  384. if eSCfg, ok := c.cfg.Scopes[s]; ok {
  385. if eSCfg.Client.Provider != nSCfg.Client.Provider ||
  386. eSCfg.Client.Address != nSCfg.Client.Address {
  387. return types.ForbiddenErrorf("cannot accept new configuration because it modifies an existing datastore client")
  388. }
  389. } else {
  390. if err := c.initScopedStore(s, nSCfg); err != nil {
  391. return err
  392. }
  393. update = true
  394. }
  395. }
  396. if !update {
  397. return nil
  398. }
  399. c.Lock()
  400. c.cfg = cfg
  401. c.Unlock()
  402. var dsConfig *discoverapi.DatastoreConfigData
  403. for scope, sCfg := range cfg.Scopes {
  404. if scope == datastore.LocalScope || !sCfg.IsValid() {
  405. continue
  406. }
  407. dsConfig = &discoverapi.DatastoreConfigData{
  408. Scope: scope,
  409. Provider: sCfg.Client.Provider,
  410. Address: sCfg.Client.Address,
  411. Config: sCfg.Client.Config,
  412. }
  413. break
  414. }
  415. if dsConfig == nil {
  416. return nil
  417. }
  418. c.drvRegistry.WalkIPAMs(func(name string, driver ipamapi.Ipam, cap *ipamapi.Capability) bool {
  419. err := driver.DiscoverNew(discoverapi.DatastoreConfig, *dsConfig)
  420. if err != nil {
  421. logrus.Errorf("Failed to set datastore in driver %s: %v", name, err)
  422. }
  423. return false
  424. })
  425. c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool {
  426. err := driver.DiscoverNew(discoverapi.DatastoreConfig, *dsConfig)
  427. if err != nil {
  428. logrus.Errorf("Failed to set datastore in driver %s: %v", name, err)
  429. }
  430. return false
  431. })
  432. return nil
  433. }
  434. func (c *controller) ID() string {
  435. return c.id
  436. }
  437. func (c *controller) BuiltinDrivers() []string {
  438. drivers := []string{}
  439. c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool {
  440. if driver.IsBuiltIn() {
  441. drivers = append(drivers, name)
  442. }
  443. return false
  444. })
  445. return drivers
  446. }
  447. func (c *controller) BuiltinIPAMDrivers() []string {
  448. drivers := []string{}
  449. c.drvRegistry.WalkIPAMs(func(name string, driver ipamapi.Ipam, cap *ipamapi.Capability) bool {
  450. if driver.IsBuiltIn() {
  451. drivers = append(drivers, name)
  452. }
  453. return false
  454. })
  455. return drivers
  456. }
  457. func (c *controller) clusterHostID() string {
  458. c.Lock()
  459. defer c.Unlock()
  460. if c.cfg == nil || c.cfg.Cluster.Address == "" {
  461. return ""
  462. }
  463. addr := strings.Split(c.cfg.Cluster.Address, ":")
  464. return addr[0]
  465. }
  466. func (c *controller) processNodeDiscovery(nodes []net.IP, add bool) {
  467. c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool {
  468. c.pushNodeDiscovery(driver, capability, nodes, add)
  469. return false
  470. })
  471. }
  472. func (c *controller) pushNodeDiscovery(d driverapi.Driver, cap driverapi.Capability, nodes []net.IP, add bool) {
  473. var self net.IP
  474. if c.cfg != nil {
  475. addr := strings.Split(c.cfg.Cluster.Address, ":")
  476. self = net.ParseIP(addr[0])
  477. // if external kvstore is not configured, try swarm-mode config
  478. if self == nil {
  479. if agent := c.getAgent(); agent != nil {
  480. self = net.ParseIP(agent.advertiseAddr)
  481. }
  482. }
  483. }
  484. if d == nil || cap.ConnectivityScope != datastore.GlobalScope || nodes == nil {
  485. return
  486. }
  487. for _, node := range nodes {
  488. nodeData := discoverapi.NodeDiscoveryData{Address: node.String(), Self: node.Equal(self)}
  489. var err error
  490. if add {
  491. err = d.DiscoverNew(discoverapi.NodeDiscovery, nodeData)
  492. } else {
  493. err = d.DiscoverDelete(discoverapi.NodeDiscovery, nodeData)
  494. }
  495. if err != nil {
  496. logrus.Debugf("discovery notification error: %v", err)
  497. }
  498. }
  499. }
  500. func (c *controller) Config() config.Config {
  501. c.Lock()
  502. defer c.Unlock()
  503. if c.cfg == nil {
  504. return config.Config{}
  505. }
  506. return *c.cfg
  507. }
  508. func (c *controller) isManager() bool {
  509. c.Lock()
  510. defer c.Unlock()
  511. if c.cfg == nil || c.cfg.Daemon.ClusterProvider == nil {
  512. return false
  513. }
  514. return c.cfg.Daemon.ClusterProvider.IsManager()
  515. }
  516. func (c *controller) isAgent() bool {
  517. c.Lock()
  518. defer c.Unlock()
  519. if c.cfg == nil || c.cfg.Daemon.ClusterProvider == nil {
  520. return false
  521. }
  522. return c.cfg.Daemon.ClusterProvider.IsAgent()
  523. }
  524. func (c *controller) isDistributedControl() bool {
  525. return !c.isManager() && !c.isAgent()
  526. }
  527. func (c *controller) GetPluginGetter() plugingetter.PluginGetter {
  528. return c.drvRegistry.GetPluginGetter()
  529. }
  530. func (c *controller) RegisterDriver(networkType string, driver driverapi.Driver, capability driverapi.Capability) error {
  531. c.agentDriverNotify(driver)
  532. return nil
  533. }
  534. // XXX This should be made driver agnostic. See comment below.
  535. const overlayDSROptionString = "dsr"
  536. // NewNetwork creates a new network of the specified network type. The options
  537. // are network specific and modeled in a generic way.
  538. func (c *controller) NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error) {
  539. var (
  540. cap *driverapi.Capability
  541. err error
  542. t *network
  543. skipCfgEpCount bool
  544. )
  545. if id != "" {
  546. c.networkLocker.Lock(id)
  547. defer c.networkLocker.Unlock(id) // nolint:errcheck
  548. if _, err = c.NetworkByID(id); err == nil {
  549. return nil, NetworkNameError(id)
  550. }
  551. }
  552. if !config.IsValidName(name) {
  553. return nil, ErrInvalidName(name)
  554. }
  555. if id == "" {
  556. id = stringid.GenerateRandomID()
  557. }
  558. defaultIpam := defaultIpamForNetworkType(networkType)
  559. // Construct the network object
  560. network := &network{
  561. name: name,
  562. networkType: networkType,
  563. generic: map[string]interface{}{netlabel.GenericData: make(map[string]string)},
  564. ipamType: defaultIpam,
  565. id: id,
  566. created: time.Now(),
  567. ctrlr: c,
  568. persist: true,
  569. drvOnce: &sync.Once{},
  570. loadBalancerMode: loadBalancerModeDefault,
  571. }
  572. network.processOptions(options...)
  573. if err = network.validateConfiguration(); err != nil {
  574. return nil, err
  575. }
  576. // Reset network types, force local scope and skip allocation and
  577. // plumbing for configuration networks. Reset of the config-only
  578. // network drivers is needed so that this special network is not
  579. // usable by old engine versions.
  580. if network.configOnly {
  581. network.scope = datastore.LocalScope
  582. network.networkType = "null"
  583. goto addToStore
  584. }
  585. _, cap, err = network.resolveDriver(network.networkType, true)
  586. if err != nil {
  587. return nil, err
  588. }
  589. if network.scope == datastore.LocalScope && cap.DataScope == datastore.GlobalScope {
  590. return nil, types.ForbiddenErrorf("cannot downgrade network scope for %s networks", networkType)
  591. }
  592. if network.ingress && cap.DataScope != datastore.GlobalScope {
  593. return nil, types.ForbiddenErrorf("Ingress network can only be global scope network")
  594. }
  595. // At this point the network scope is still unknown if not set by user
  596. if (cap.DataScope == datastore.GlobalScope || network.scope == datastore.SwarmScope) &&
  597. !c.isDistributedControl() && !network.dynamic {
  598. if c.isManager() {
  599. // For non-distributed controlled environment, globalscoped non-dynamic networks are redirected to Manager
  600. return nil, ManagerRedirectError(name)
  601. }
  602. return nil, types.ForbiddenErrorf("Cannot create a multi-host network from a worker node. Please create the network from a manager node.")
  603. }
  604. if network.scope == datastore.SwarmScope && c.isDistributedControl() {
  605. return nil, types.ForbiddenErrorf("cannot create a swarm scoped network when swarm is not active")
  606. }
  607. // Make sure we have a driver available for this network type
  608. // before we allocate anything.
  609. if _, err := network.driver(true); err != nil {
  610. return nil, err
  611. }
  612. // From this point on, we need the network specific configuration,
  613. // which may come from a configuration-only network
  614. if network.configFrom != "" {
  615. t, err = c.getConfigNetwork(network.configFrom)
  616. if err != nil {
  617. return nil, types.NotFoundErrorf("configuration network %q does not exist", network.configFrom)
  618. }
  619. if err = t.applyConfigurationTo(network); err != nil {
  620. return nil, types.InternalErrorf("Failed to apply configuration: %v", err)
  621. }
  622. network.generic[netlabel.Internal] = network.internal
  623. defer func() {
  624. if err == nil && !skipCfgEpCount {
  625. if err := t.getEpCnt().IncEndpointCnt(); err != nil {
  626. logrus.Warnf("Failed to update reference count for configuration network %q on creation of network %q: %v",
  627. t.Name(), network.Name(), err)
  628. }
  629. }
  630. }()
  631. }
  632. err = network.ipamAllocate()
  633. if err != nil {
  634. return nil, err
  635. }
  636. defer func() {
  637. if err != nil {
  638. network.ipamRelease()
  639. }
  640. }()
  641. err = c.addNetwork(network)
  642. if err != nil {
  643. if _, ok := err.(types.MaskableError); ok { // nolint:gosimple
  644. // This error can be ignored and set this boolean
  645. // value to skip a refcount increment for configOnly networks
  646. skipCfgEpCount = true
  647. } else {
  648. return nil, err
  649. }
  650. }
  651. defer func() {
  652. if err != nil {
  653. if e := network.deleteNetwork(); e != nil {
  654. logrus.Warnf("couldn't roll back driver network on network %s creation failure: %v", network.name, err)
  655. }
  656. }
  657. }()
  658. // XXX If the driver type is "overlay" check the options for DSR
  659. // being set. If so, set the network's load balancing mode to DSR.
  660. // This should really be done in a network option, but due to
  661. // time pressure to get this in without adding changes to moby,
  662. // swarm and CLI, it is being implemented as a driver-specific
  663. // option. Unfortunately, drivers can't influence the core
  664. // "libnetwork.network" data type. Hence we need this hack code
  665. // to implement in this manner.
  666. if gval, ok := network.generic[netlabel.GenericData]; ok && network.networkType == "overlay" {
  667. optMap := gval.(map[string]string)
  668. if _, ok := optMap[overlayDSROptionString]; ok {
  669. network.loadBalancerMode = loadBalancerModeDSR
  670. }
  671. }
  672. addToStore:
  673. // First store the endpoint count, then the network. To avoid to
  674. // end up with a datastore containing a network and not an epCnt,
  675. // in case of an ungraceful shutdown during this function call.
  676. epCnt := &endpointCnt{n: network}
  677. if err = c.updateToStore(epCnt); err != nil {
  678. return nil, err
  679. }
  680. defer func() {
  681. if err != nil {
  682. if e := c.deleteFromStore(epCnt); e != nil {
  683. logrus.Warnf("could not rollback from store, epCnt %v on failure (%v): %v", epCnt, err, e)
  684. }
  685. }
  686. }()
  687. network.epCnt = epCnt
  688. if err = c.updateToStore(network); err != nil {
  689. return nil, err
  690. }
  691. defer func() {
  692. if err != nil {
  693. if e := c.deleteFromStore(network); e != nil {
  694. logrus.Warnf("could not rollback from store, network %v on failure (%v): %v", network, err, e)
  695. }
  696. }
  697. }()
  698. if network.configOnly {
  699. return network, nil
  700. }
  701. joinCluster(network)
  702. defer func() {
  703. if err != nil {
  704. network.cancelDriverWatches()
  705. if e := network.leaveCluster(); e != nil {
  706. logrus.Warnf("Failed to leave agent cluster on network %s on failure (%v): %v", network.name, err, e)
  707. }
  708. }
  709. }()
  710. if network.hasLoadBalancerEndpoint() {
  711. if err = network.createLoadBalancerSandbox(); err != nil {
  712. return nil, err
  713. }
  714. }
  715. if !c.isDistributedControl() {
  716. c.Lock()
  717. arrangeIngressFilterRule()
  718. c.Unlock()
  719. }
  720. arrangeUserFilterRule()
  721. return network, nil
  722. }
  723. var joinCluster NetworkWalker = func(nw Network) bool {
  724. n := nw.(*network)
  725. if n.configOnly {
  726. return false
  727. }
  728. if err := n.joinCluster(); err != nil {
  729. logrus.Errorf("Failed to join network %s (%s) into agent cluster: %v", n.Name(), n.ID(), err)
  730. }
  731. n.addDriverWatches()
  732. return false
  733. }
  734. func (c *controller) reservePools() {
  735. networks, err := c.getNetworksForScope(datastore.LocalScope)
  736. if err != nil {
  737. logrus.Warnf("Could not retrieve networks from local store during ipam allocation for existing networks: %v", err)
  738. return
  739. }
  740. for _, n := range networks {
  741. if n.configOnly {
  742. continue
  743. }
  744. if !doReplayPoolReserve(n) {
  745. continue
  746. }
  747. // Construct pseudo configs for the auto IP case
  748. autoIPv4 := (len(n.ipamV4Config) == 0 || (len(n.ipamV4Config) == 1 && n.ipamV4Config[0].PreferredPool == "")) && len(n.ipamV4Info) > 0
  749. autoIPv6 := (len(n.ipamV6Config) == 0 || (len(n.ipamV6Config) == 1 && n.ipamV6Config[0].PreferredPool == "")) && len(n.ipamV6Info) > 0
  750. if autoIPv4 {
  751. n.ipamV4Config = []*IpamConf{{PreferredPool: n.ipamV4Info[0].Pool.String()}}
  752. }
  753. if n.enableIPv6 && autoIPv6 {
  754. n.ipamV6Config = []*IpamConf{{PreferredPool: n.ipamV6Info[0].Pool.String()}}
  755. }
  756. // Account current network gateways
  757. for i, c := range n.ipamV4Config {
  758. if c.Gateway == "" && n.ipamV4Info[i].Gateway != nil {
  759. c.Gateway = n.ipamV4Info[i].Gateway.IP.String()
  760. }
  761. }
  762. if n.enableIPv6 {
  763. for i, c := range n.ipamV6Config {
  764. if c.Gateway == "" && n.ipamV6Info[i].Gateway != nil {
  765. c.Gateway = n.ipamV6Info[i].Gateway.IP.String()
  766. }
  767. }
  768. }
  769. // Reserve pools
  770. if err := n.ipamAllocate(); err != nil {
  771. logrus.Warnf("Failed to allocate ipam pool(s) for network %q (%s): %v", n.Name(), n.ID(), err)
  772. }
  773. // Reserve existing endpoints' addresses
  774. ipam, _, err := n.getController().getIPAMDriver(n.ipamType)
  775. if err != nil {
  776. logrus.Warnf("Failed to retrieve ipam driver for network %q (%s) during address reservation", n.Name(), n.ID())
  777. continue
  778. }
  779. epl, err := n.getEndpointsFromStore()
  780. if err != nil {
  781. logrus.Warnf("Failed to retrieve list of current endpoints on network %q (%s)", n.Name(), n.ID())
  782. continue
  783. }
  784. for _, ep := range epl {
  785. if ep.Iface() == nil {
  786. logrus.Warnf("endpoint interface is empty for %q (%s)", ep.Name(), ep.ID())
  787. continue
  788. }
  789. if err := ep.assignAddress(ipam, true, ep.Iface().AddressIPv6() != nil); err != nil {
  790. logrus.Warnf("Failed to reserve current address for endpoint %q (%s) on network %q (%s)",
  791. ep.Name(), ep.ID(), n.Name(), n.ID())
  792. }
  793. }
  794. }
  795. }
  796. func doReplayPoolReserve(n *network) bool {
  797. _, caps, err := n.getController().getIPAMDriver(n.ipamType)
  798. if err != nil {
  799. logrus.Warnf("Failed to retrieve ipam driver for network %q (%s): %v", n.Name(), n.ID(), err)
  800. return false
  801. }
  802. return caps.RequiresRequestReplay
  803. }
  804. func (c *controller) addNetwork(n *network) error {
  805. d, err := n.driver(true)
  806. if err != nil {
  807. return err
  808. }
  809. // Create the network
  810. if err := d.CreateNetwork(n.id, n.generic, n, n.getIPData(4), n.getIPData(6)); err != nil {
  811. return err
  812. }
  813. n.startResolver()
  814. return nil
  815. }
  816. func (c *controller) Networks() []Network {
  817. var list []Network
  818. for _, n := range c.getNetworksFromStore() {
  819. if n.inDelete {
  820. continue
  821. }
  822. list = append(list, n)
  823. }
  824. return list
  825. }
  826. func (c *controller) WalkNetworks(walker NetworkWalker) {
  827. for _, n := range c.Networks() {
  828. if walker(n) {
  829. return
  830. }
  831. }
  832. }
  833. func (c *controller) NetworkByName(name string) (Network, error) {
  834. if name == "" {
  835. return nil, ErrInvalidName(name)
  836. }
  837. var n Network
  838. s := func(current Network) bool {
  839. if current.Name() == name {
  840. n = current
  841. return true
  842. }
  843. return false
  844. }
  845. c.WalkNetworks(s)
  846. if n == nil {
  847. return nil, ErrNoSuchNetwork(name)
  848. }
  849. return n, nil
  850. }
  851. func (c *controller) NetworkByID(id string) (Network, error) {
  852. if id == "" {
  853. return nil, ErrInvalidID(id)
  854. }
  855. n, err := c.getNetworkFromStore(id)
  856. if err != nil {
  857. return nil, ErrNoSuchNetwork(id)
  858. }
  859. return n, nil
  860. }
  861. // NewSandbox creates a new sandbox for the passed container id
  862. func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error) {
  863. if containerID == "" {
  864. return nil, types.BadRequestErrorf("invalid container ID")
  865. }
  866. var sb *sandbox
  867. c.Lock()
  868. for _, s := range c.sandboxes {
  869. if s.containerID == containerID {
  870. // If not a stub, then we already have a complete sandbox.
  871. if !s.isStub {
  872. sbID := s.ID()
  873. c.Unlock()
  874. return nil, types.ForbiddenErrorf("container %s is already present in sandbox %s", containerID, sbID)
  875. }
  876. // We already have a stub sandbox from the
  877. // store. Make use of it so that we don't lose
  878. // the endpoints from store but reset the
  879. // isStub flag.
  880. sb = s
  881. sb.isStub = false
  882. break
  883. }
  884. }
  885. c.Unlock()
  886. sandboxID := stringid.GenerateRandomID()
  887. if runtime.GOOS == "windows" {
  888. sandboxID = containerID
  889. }
  890. // Create sandbox and process options first. Key generation depends on an option
  891. if sb == nil {
  892. sb = &sandbox{
  893. id: sandboxID,
  894. containerID: containerID,
  895. endpoints: []*endpoint{},
  896. epPriority: map[string]int{},
  897. populatedEndpoints: map[string]struct{}{},
  898. config: containerConfig{},
  899. controller: c,
  900. extDNS: []extDNSEntry{},
  901. }
  902. }
  903. sb.processOptions(options...)
  904. c.Lock()
  905. if sb.ingress && c.ingressSandbox != nil {
  906. c.Unlock()
  907. return nil, types.ForbiddenErrorf("ingress sandbox already present")
  908. }
  909. if sb.ingress {
  910. c.ingressSandbox = sb
  911. sb.config.hostsPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/hosts")
  912. sb.config.resolvConfPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/resolv.conf")
  913. sb.id = "ingress_sbox"
  914. } else if sb.loadBalancerNID != "" {
  915. sb.id = "lb_" + sb.loadBalancerNID
  916. }
  917. c.Unlock()
  918. var err error
  919. defer func() {
  920. if err != nil {
  921. c.Lock()
  922. if sb.ingress {
  923. c.ingressSandbox = nil
  924. }
  925. c.Unlock()
  926. }
  927. }()
  928. if err = sb.setupResolutionFiles(); err != nil {
  929. return nil, err
  930. }
  931. if sb.config.useDefaultSandBox {
  932. c.sboxOnce.Do(func() {
  933. c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
  934. })
  935. if err != nil {
  936. c.sboxOnce = sync.Once{}
  937. return nil, fmt.Errorf("failed to create default sandbox: %v", err)
  938. }
  939. sb.osSbox = c.defOsSbox
  940. }
  941. if sb.osSbox == nil && !sb.config.useExternalKey {
  942. if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil {
  943. return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
  944. }
  945. }
  946. if sb.osSbox != nil {
  947. // Apply operating specific knobs on the load balancer sandbox
  948. sb.osSbox.ApplyOSTweaks(sb.oslTypes)
  949. }
  950. c.Lock()
  951. c.sandboxes[sb.id] = sb
  952. c.Unlock()
  953. defer func() {
  954. if err != nil {
  955. c.Lock()
  956. delete(c.sandboxes, sb.id)
  957. c.Unlock()
  958. }
  959. }()
  960. err = sb.storeUpdate()
  961. if err != nil {
  962. return nil, fmt.Errorf("failed to update the store state of sandbox: %v", err)
  963. }
  964. return sb, nil
  965. }
  966. func (c *controller) Sandboxes() []Sandbox {
  967. c.Lock()
  968. defer c.Unlock()
  969. list := make([]Sandbox, 0, len(c.sandboxes))
  970. for _, s := range c.sandboxes {
  971. // Hide stub sandboxes from libnetwork users
  972. if s.isStub {
  973. continue
  974. }
  975. list = append(list, s)
  976. }
  977. return list
  978. }
  979. func (c *controller) WalkSandboxes(walker SandboxWalker) {
  980. for _, sb := range c.Sandboxes() {
  981. if walker(sb) {
  982. return
  983. }
  984. }
  985. }
  986. func (c *controller) SandboxByID(id string) (Sandbox, error) {
  987. if id == "" {
  988. return nil, ErrInvalidID(id)
  989. }
  990. c.Lock()
  991. s, ok := c.sandboxes[id]
  992. c.Unlock()
  993. if !ok {
  994. return nil, types.NotFoundErrorf("sandbox %s not found", id)
  995. }
  996. return s, nil
  997. }
  998. // SandboxDestroy destroys a sandbox given a container ID
  999. func (c *controller) SandboxDestroy(id string) error {
  1000. var sb *sandbox
  1001. c.Lock()
  1002. for _, s := range c.sandboxes {
  1003. if s.containerID == id {
  1004. sb = s
  1005. break
  1006. }
  1007. }
  1008. c.Unlock()
  1009. // It is not an error if sandbox is not available
  1010. if sb == nil {
  1011. return nil
  1012. }
  1013. return sb.Delete()
  1014. }
  1015. // SandboxContainerWalker returns a Sandbox Walker function which looks for an existing Sandbox with the passed containerID
  1016. func SandboxContainerWalker(out *Sandbox, containerID string) SandboxWalker {
  1017. return func(sb Sandbox) bool {
  1018. if sb.ContainerID() == containerID {
  1019. *out = sb
  1020. return true
  1021. }
  1022. return false
  1023. }
  1024. }
  1025. // SandboxKeyWalker returns a Sandbox Walker function which looks for an existing Sandbox with the passed key
  1026. func SandboxKeyWalker(out *Sandbox, key string) SandboxWalker {
  1027. return func(sb Sandbox) bool {
  1028. if sb.Key() == key {
  1029. *out = sb
  1030. return true
  1031. }
  1032. return false
  1033. }
  1034. }
  1035. func (c *controller) loadDriver(networkType string) error {
  1036. var err error
  1037. if pg := c.GetPluginGetter(); pg != nil {
  1038. _, err = pg.Get(networkType, driverapi.NetworkPluginEndpointType, plugingetter.Lookup)
  1039. } else {
  1040. _, err = plugins.Get(networkType, driverapi.NetworkPluginEndpointType)
  1041. }
  1042. if err != nil {
  1043. if errors.Cause(err) == plugins.ErrNotFound {
  1044. return types.NotFoundErrorf(err.Error())
  1045. }
  1046. return err
  1047. }
  1048. return nil
  1049. }
  1050. func (c *controller) loadIPAMDriver(name string) error {
  1051. var err error
  1052. if pg := c.GetPluginGetter(); pg != nil {
  1053. _, err = pg.Get(name, ipamapi.PluginEndpointType, plugingetter.Lookup)
  1054. } else {
  1055. _, err = plugins.Get(name, ipamapi.PluginEndpointType)
  1056. }
  1057. if err != nil {
  1058. if errors.Cause(err) == plugins.ErrNotFound {
  1059. return types.NotFoundErrorf(err.Error())
  1060. }
  1061. return err
  1062. }
  1063. return nil
  1064. }
  1065. func (c *controller) getIPAMDriver(name string) (ipamapi.Ipam, *ipamapi.Capability, error) {
  1066. id, cap := c.drvRegistry.IPAM(name)
  1067. if id == nil {
  1068. // Might be a plugin name. Try loading it
  1069. if err := c.loadIPAMDriver(name); err != nil {
  1070. return nil, nil, err
  1071. }
  1072. // Now that we resolved the plugin, try again looking up the registry
  1073. id, cap = c.drvRegistry.IPAM(name)
  1074. if id == nil {
  1075. return nil, nil, types.BadRequestErrorf("invalid ipam driver: %q", name)
  1076. }
  1077. }
  1078. return id, cap, nil
  1079. }
  1080. func (c *controller) Stop() {
  1081. c.closeStores()
  1082. c.stopExternalKeyListener()
  1083. osl.GC()
  1084. }
  1085. // StartDiagnostic start the network dias mode
  1086. func (c *controller) StartDiagnostic(port int) {
  1087. c.Lock()
  1088. if !c.DiagnosticServer.IsDiagnosticEnabled() {
  1089. c.DiagnosticServer.EnableDiagnostic("127.0.0.1", port)
  1090. }
  1091. c.Unlock()
  1092. }
  1093. // StopDiagnostic start the network dias mode
  1094. func (c *controller) StopDiagnostic() {
  1095. c.Lock()
  1096. if c.DiagnosticServer.IsDiagnosticEnabled() {
  1097. c.DiagnosticServer.DisableDiagnostic()
  1098. }
  1099. c.Unlock()
  1100. }
  1101. // IsDiagnosticEnabled returns true if the dias is enabled
  1102. func (c *controller) IsDiagnosticEnabled() bool {
  1103. c.Lock()
  1104. defer c.Unlock()
  1105. return c.DiagnosticServer.IsDiagnosticEnabled()
  1106. }
  1107. func (c *controller) iptablesEnabled() bool {
  1108. c.Lock()
  1109. defer c.Unlock()
  1110. if c.cfg == nil {
  1111. return false
  1112. }
  1113. // parse map cfg["bridge"]["generic"]["EnableIPTable"]
  1114. cfgBridge, ok := c.cfg.Daemon.DriverCfg["bridge"].(map[string]interface{})
  1115. if !ok {
  1116. return false
  1117. }
  1118. cfgGeneric, ok := cfgBridge[netlabel.GenericData].(options.Generic)
  1119. if !ok {
  1120. return false
  1121. }
  1122. enabled, ok := cfgGeneric["EnableIPTables"].(bool)
  1123. if !ok {
  1124. // unless user explicitly stated, assume iptable is enabled
  1125. enabled = true
  1126. }
  1127. return enabled
  1128. }