controller.go 30 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177
  1. /*
  2. Package libnetwork provides the basic functionality and extension points to
  3. create network namespaces and allocate interfaces for containers to use.
  4. networkType := "bridge"
  5. // Create a new controller instance
  6. driverOptions := options.Generic{}
  7. genericOption := make(map[string]interface{})
  8. genericOption[netlabel.GenericData] = driverOptions
  9. controller, err := libnetwork.New(config.OptionDriverConfig(networkType, genericOption))
  10. if err != nil {
  11. return
  12. }
  13. // Create a network for containers to join.
  14. // NewNetwork accepts Variadic optional arguments that libnetwork and Drivers can make use of
  15. network, err := controller.NewNetwork(networkType, "network1", "")
  16. if err != nil {
  17. return
  18. }
  19. // For each new container: allocate IP and interfaces. The returned network
  20. // settings will be used for container infos (inspect and such), as well as
  21. // iptables rules for port publishing. This info is contained or accessible
  22. // from the returned endpoint.
  23. ep, err := network.CreateEndpoint("Endpoint1")
  24. if err != nil {
  25. return
  26. }
  27. // Create the sandbox for the container.
  28. // NewSandbox accepts Variadic optional arguments which libnetwork can use.
  29. sbx, err := controller.NewSandbox("container1",
  30. libnetwork.OptionHostname("test"),
  31. libnetwork.OptionDomainname("docker.io"))
  32. // A sandbox can join the endpoint via the join api.
  33. err = ep.Join(sbx)
  34. if err != nil {
  35. return
  36. }
  37. */
  38. package libnetwork
  39. import (
  40. "container/heap"
  41. "fmt"
  42. "net"
  43. "strings"
  44. "sync"
  45. "time"
  46. "github.com/Sirupsen/logrus"
  47. "github.com/docker/docker/pkg/discovery"
  48. "github.com/docker/docker/pkg/locker"
  49. "github.com/docker/docker/pkg/plugingetter"
  50. "github.com/docker/docker/pkg/plugins"
  51. "github.com/docker/docker/pkg/stringid"
  52. "github.com/docker/libnetwork/cluster"
  53. "github.com/docker/libnetwork/config"
  54. "github.com/docker/libnetwork/datastore"
  55. "github.com/docker/libnetwork/discoverapi"
  56. "github.com/docker/libnetwork/driverapi"
  57. "github.com/docker/libnetwork/drvregistry"
  58. "github.com/docker/libnetwork/hostdiscovery"
  59. "github.com/docker/libnetwork/ipamapi"
  60. "github.com/docker/libnetwork/netlabel"
  61. "github.com/docker/libnetwork/osl"
  62. "github.com/docker/libnetwork/types"
  63. )
  64. // NetworkController provides the interface for controller instance which manages
  65. // networks.
  66. type NetworkController interface {
  67. // ID provides a unique identity for the controller
  68. ID() string
  69. // BuiltinDrivers returns list of builtin drivers
  70. BuiltinDrivers() []string
  71. // BuiltinIPAMDrivers returns list of builtin ipam drivers
  72. BuiltinIPAMDrivers() []string
  73. // Config method returns the bootup configuration for the controller
  74. Config() config.Config
  75. // Create a new network. The options parameter carries network specific options.
  76. NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error)
  77. // Networks returns the list of Network(s) managed by this controller.
  78. Networks() []Network
  79. // WalkNetworks uses the provided function to walk the Network(s) managed by this controller.
  80. WalkNetworks(walker NetworkWalker)
  81. // NetworkByName returns the Network which has the passed name. If not found, the error ErrNoSuchNetwork is returned.
  82. NetworkByName(name string) (Network, error)
  83. // NetworkByID returns the Network which has the passed id. If not found, the error ErrNoSuchNetwork is returned.
  84. NetworkByID(id string) (Network, error)
  85. // NewSandbox creates a new network sandbox for the passed container id
  86. NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error)
  87. // Sandboxes returns the list of Sandbox(s) managed by this controller.
  88. Sandboxes() []Sandbox
  89. // WalkSandboxes uses the provided function to walk the Sandbox(s) managed by this controller.
  90. WalkSandboxes(walker SandboxWalker)
  91. // SandboxByID returns the Sandbox which has the passed id. If not found, a types.NotFoundError is returned.
  92. SandboxByID(id string) (Sandbox, error)
  93. // SandboxDestroy destroys a sandbox given a container ID
  94. SandboxDestroy(id string) error
  95. // Stop network controller
  96. Stop()
  97. // ReloadCondfiguration updates the controller configuration
  98. ReloadConfiguration(cfgOptions ...config.Option) error
  99. // SetClusterProvider sets cluster provider
  100. SetClusterProvider(provider cluster.Provider)
  101. // Wait for agent initialization complete in libnetwork controller
  102. AgentInitWait()
  103. // Wait for agent to stop if running
  104. AgentStopWait()
  105. // SetKeys configures the encryption key for gossip and overlay data path
  106. SetKeys(keys []*types.EncryptionKey) error
  107. }
  108. // NetworkWalker is a client provided function which will be used to walk the Networks.
  109. // When the function returns true, the walk will stop.
  110. type NetworkWalker func(nw Network) bool
  111. // SandboxWalker is a client provided function which will be used to walk the Sandboxes.
  112. // When the function returns true, the walk will stop.
  113. type SandboxWalker func(sb Sandbox) bool
  114. type sandboxTable map[string]*sandbox
  115. type controller struct {
  116. id string
  117. drvRegistry *drvregistry.DrvRegistry
  118. sandboxes sandboxTable
  119. cfg *config.Config
  120. stores []datastore.DataStore
  121. discovery hostdiscovery.HostDiscovery
  122. extKeyListener net.Listener
  123. watchCh chan *endpoint
  124. unWatchCh chan *endpoint
  125. svcRecords map[string]svcInfo
  126. nmap map[string]*netWatch
  127. serviceBindings map[serviceKey]*service
  128. defOsSbox osl.Sandbox
  129. ingressSandbox *sandbox
  130. sboxOnce sync.Once
  131. agent *agent
  132. networkLocker *locker.Locker
  133. agentInitDone chan struct{}
  134. agentStopDone chan struct{}
  135. keys []*types.EncryptionKey
  136. clusterConfigAvailable bool
  137. sync.Mutex
  138. }
  139. type initializer struct {
  140. fn drvregistry.InitFunc
  141. ntype string
  142. }
  143. // New creates a new instance of network controller.
  144. func New(cfgOptions ...config.Option) (NetworkController, error) {
  145. c := &controller{
  146. id: stringid.GenerateRandomID(),
  147. cfg: config.ParseConfigOptions(cfgOptions...),
  148. sandboxes: sandboxTable{},
  149. svcRecords: make(map[string]svcInfo),
  150. serviceBindings: make(map[serviceKey]*service),
  151. agentInitDone: make(chan struct{}),
  152. networkLocker: locker.New(),
  153. }
  154. if err := c.initStores(); err != nil {
  155. return nil, err
  156. }
  157. drvRegistry, err := drvregistry.New(c.getStore(datastore.LocalScope), c.getStore(datastore.GlobalScope), c.RegisterDriver, nil, c.cfg.PluginGetter)
  158. if err != nil {
  159. return nil, err
  160. }
  161. for _, i := range getInitializers(c.cfg.Daemon.Experimental) {
  162. var dcfg map[string]interface{}
  163. // External plugins don't need config passed through daemon. They can
  164. // bootstrap themselves
  165. if i.ntype != "remote" {
  166. dcfg = c.makeDriverConfig(i.ntype)
  167. }
  168. if err := drvRegistry.AddDriver(i.ntype, i.fn, dcfg); err != nil {
  169. return nil, err
  170. }
  171. }
  172. if err = initIPAMDrivers(drvRegistry, nil, c.getStore(datastore.GlobalScope)); err != nil {
  173. return nil, err
  174. }
  175. c.drvRegistry = drvRegistry
  176. if c.cfg != nil && c.cfg.Cluster.Watcher != nil {
  177. if err := c.initDiscovery(c.cfg.Cluster.Watcher); err != nil {
  178. // Failing to initialize discovery is a bad situation to be in.
  179. // But it cannot fail creating the Controller
  180. logrus.Errorf("Failed to Initialize Discovery : %v", err)
  181. }
  182. }
  183. c.WalkNetworks(populateSpecial)
  184. // Reserve pools first before doing cleanup. Otherwise the
  185. // cleanups of endpoint/network and sandbox below will
  186. // generate many unnecessary warnings
  187. c.reservePools()
  188. // Cleanup resources
  189. c.sandboxCleanup(c.cfg.ActiveSandboxes)
  190. c.cleanupLocalEndpoints()
  191. c.networkCleanup()
  192. if err := c.startExternalKeyListener(); err != nil {
  193. return nil, err
  194. }
  195. return c, nil
  196. }
  197. func (c *controller) SetClusterProvider(provider cluster.Provider) {
  198. c.Lock()
  199. c.cfg.Daemon.ClusterProvider = provider
  200. disableProviderCh := c.cfg.Daemon.DisableProvider
  201. c.Unlock()
  202. if provider != nil {
  203. go c.clusterAgentInit()
  204. } else {
  205. disableProviderCh <- struct{}{}
  206. }
  207. }
  208. func isValidClusteringIP(addr string) bool {
  209. return addr != "" && !net.ParseIP(addr).IsLoopback() && !net.ParseIP(addr).IsUnspecified()
  210. }
  211. // libnetwork side of agent depends on the keys. On the first receipt of
  212. // keys setup the agent. For subsequent key set handle the key change
  213. func (c *controller) SetKeys(keys []*types.EncryptionKey) error {
  214. c.Lock()
  215. existingKeys := c.keys
  216. clusterConfigAvailable := c.clusterConfigAvailable
  217. agent := c.agent
  218. c.Unlock()
  219. subsysKeys := make(map[string]int)
  220. for _, key := range keys {
  221. if key.Subsystem != subsysGossip &&
  222. key.Subsystem != subsysIPSec {
  223. return fmt.Errorf("key received for unrecognized subsystem")
  224. }
  225. subsysKeys[key.Subsystem]++
  226. }
  227. for s, count := range subsysKeys {
  228. if count != keyringSize {
  229. return fmt.Errorf("incorrect number of keys for subsystem %v", s)
  230. }
  231. }
  232. if len(existingKeys) == 0 {
  233. c.Lock()
  234. c.keys = keys
  235. c.Unlock()
  236. if agent != nil {
  237. return (fmt.Errorf("libnetwork agent setup without keys"))
  238. }
  239. if clusterConfigAvailable {
  240. return c.agentSetup()
  241. }
  242. logrus.Debug("received encryption keys before cluster config")
  243. return nil
  244. }
  245. if agent == nil {
  246. c.Lock()
  247. c.keys = keys
  248. c.Unlock()
  249. return nil
  250. }
  251. return c.handleKeyChange(keys)
  252. }
  253. func (c *controller) getAgent() *agent {
  254. c.Lock()
  255. defer c.Unlock()
  256. return c.agent
  257. }
  258. func (c *controller) clusterAgentInit() {
  259. clusterProvider := c.cfg.Daemon.ClusterProvider
  260. for {
  261. select {
  262. case <-clusterProvider.ListenClusterEvents():
  263. if !c.isDistributedControl() {
  264. c.Lock()
  265. c.clusterConfigAvailable = true
  266. keys := c.keys
  267. c.Unlock()
  268. // agent initialization needs encryption keys and bind/remote IP which
  269. // comes from the daemon cluster events
  270. if len(keys) > 0 {
  271. c.agentSetup()
  272. }
  273. }
  274. case <-c.cfg.Daemon.DisableProvider:
  275. c.Lock()
  276. c.clusterConfigAvailable = false
  277. c.agentInitDone = make(chan struct{})
  278. c.keys = nil
  279. c.Unlock()
  280. // We are leaving the cluster. Make sure we
  281. // close the gossip so that we stop all
  282. // incoming gossip updates before cleaning up
  283. // any remaining service bindings. But before
  284. // deleting the networks since the networks
  285. // should still be present when cleaning up
  286. // service bindings
  287. c.agentClose()
  288. c.cleanupServiceBindings("")
  289. c.Lock()
  290. if c.agentStopDone != nil {
  291. close(c.agentStopDone)
  292. c.agentStopDone = nil
  293. }
  294. c.Unlock()
  295. return
  296. }
  297. }
  298. }
  299. // AgentInitWait waits for agent initialization to be completed in the
  300. // controller.
  301. func (c *controller) AgentInitWait() {
  302. c.Lock()
  303. agentInitDone := c.agentInitDone
  304. c.Unlock()
  305. if agentInitDone != nil {
  306. <-agentInitDone
  307. }
  308. }
  309. func (c *controller) AgentStopWait() {
  310. c.Lock()
  311. agentStopDone := c.agentStopDone
  312. c.Unlock()
  313. if agentStopDone != nil {
  314. <-agentStopDone
  315. }
  316. }
  317. func (c *controller) makeDriverConfig(ntype string) map[string]interface{} {
  318. if c.cfg == nil {
  319. return nil
  320. }
  321. config := make(map[string]interface{})
  322. for _, label := range c.cfg.Daemon.Labels {
  323. if !strings.HasPrefix(netlabel.Key(label), netlabel.DriverPrefix+"."+ntype) {
  324. continue
  325. }
  326. config[netlabel.Key(label)] = netlabel.Value(label)
  327. }
  328. drvCfg, ok := c.cfg.Daemon.DriverCfg[ntype]
  329. if ok {
  330. for k, v := range drvCfg.(map[string]interface{}) {
  331. config[k] = v
  332. }
  333. }
  334. for k, v := range c.cfg.Scopes {
  335. if !v.IsValid() {
  336. continue
  337. }
  338. config[netlabel.MakeKVClient(k)] = discoverapi.DatastoreConfigData{
  339. Scope: k,
  340. Provider: v.Client.Provider,
  341. Address: v.Client.Address,
  342. Config: v.Client.Config,
  343. }
  344. }
  345. return config
  346. }
  347. var procReloadConfig = make(chan (bool), 1)
  348. func (c *controller) ReloadConfiguration(cfgOptions ...config.Option) error {
  349. procReloadConfig <- true
  350. defer func() { <-procReloadConfig }()
  351. // For now we accept the configuration reload only as a mean to provide a global store config after boot.
  352. // Refuse the configuration if it alters an existing datastore client configuration.
  353. update := false
  354. cfg := config.ParseConfigOptions(cfgOptions...)
  355. for s := range c.cfg.Scopes {
  356. if _, ok := cfg.Scopes[s]; !ok {
  357. return types.ForbiddenErrorf("cannot accept new configuration because it removes an existing datastore client")
  358. }
  359. }
  360. for s, nSCfg := range cfg.Scopes {
  361. if eSCfg, ok := c.cfg.Scopes[s]; ok {
  362. if eSCfg.Client.Provider != nSCfg.Client.Provider ||
  363. eSCfg.Client.Address != nSCfg.Client.Address {
  364. return types.ForbiddenErrorf("cannot accept new configuration because it modifies an existing datastore client")
  365. }
  366. } else {
  367. if err := c.initScopedStore(s, nSCfg); err != nil {
  368. return err
  369. }
  370. update = true
  371. }
  372. }
  373. if !update {
  374. return nil
  375. }
  376. c.Lock()
  377. c.cfg = cfg
  378. c.Unlock()
  379. var dsConfig *discoverapi.DatastoreConfigData
  380. for scope, sCfg := range cfg.Scopes {
  381. if scope == datastore.LocalScope || !sCfg.IsValid() {
  382. continue
  383. }
  384. dsConfig = &discoverapi.DatastoreConfigData{
  385. Scope: scope,
  386. Provider: sCfg.Client.Provider,
  387. Address: sCfg.Client.Address,
  388. Config: sCfg.Client.Config,
  389. }
  390. break
  391. }
  392. if dsConfig == nil {
  393. return nil
  394. }
  395. c.drvRegistry.WalkIPAMs(func(name string, driver ipamapi.Ipam, cap *ipamapi.Capability) bool {
  396. err := driver.DiscoverNew(discoverapi.DatastoreConfig, *dsConfig)
  397. if err != nil {
  398. logrus.Errorf("Failed to set datastore in driver %s: %v", name, err)
  399. }
  400. return false
  401. })
  402. c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool {
  403. err := driver.DiscoverNew(discoverapi.DatastoreConfig, *dsConfig)
  404. if err != nil {
  405. logrus.Errorf("Failed to set datastore in driver %s: %v", name, err)
  406. }
  407. return false
  408. })
  409. if c.discovery == nil && c.cfg.Cluster.Watcher != nil {
  410. if err := c.initDiscovery(c.cfg.Cluster.Watcher); err != nil {
  411. logrus.Errorf("Failed to Initialize Discovery after configuration update: %v", err)
  412. }
  413. }
  414. return nil
  415. }
  416. func (c *controller) ID() string {
  417. return c.id
  418. }
  419. func (c *controller) BuiltinDrivers() []string {
  420. drivers := []string{}
  421. c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool {
  422. if driver.IsBuiltIn() {
  423. drivers = append(drivers, name)
  424. }
  425. return false
  426. })
  427. return drivers
  428. }
  429. func (c *controller) BuiltinIPAMDrivers() []string {
  430. drivers := []string{}
  431. c.drvRegistry.WalkIPAMs(func(name string, driver ipamapi.Ipam, cap *ipamapi.Capability) bool {
  432. if driver.IsBuiltIn() {
  433. drivers = append(drivers, name)
  434. }
  435. return false
  436. })
  437. return drivers
  438. }
  439. func (c *controller) validateHostDiscoveryConfig() bool {
  440. if c.cfg == nil || c.cfg.Cluster.Discovery == "" || c.cfg.Cluster.Address == "" {
  441. return false
  442. }
  443. return true
  444. }
  445. func (c *controller) clusterHostID() string {
  446. c.Lock()
  447. defer c.Unlock()
  448. if c.cfg == nil || c.cfg.Cluster.Address == "" {
  449. return ""
  450. }
  451. addr := strings.Split(c.cfg.Cluster.Address, ":")
  452. return addr[0]
  453. }
  454. func (c *controller) isNodeAlive(node string) bool {
  455. if c.discovery == nil {
  456. return false
  457. }
  458. nodes := c.discovery.Fetch()
  459. for _, n := range nodes {
  460. if n.String() == node {
  461. return true
  462. }
  463. }
  464. return false
  465. }
  466. func (c *controller) initDiscovery(watcher discovery.Watcher) error {
  467. if c.cfg == nil {
  468. return fmt.Errorf("discovery initialization requires a valid configuration")
  469. }
  470. c.discovery = hostdiscovery.NewHostDiscovery(watcher)
  471. return c.discovery.Watch(c.activeCallback, c.hostJoinCallback, c.hostLeaveCallback)
  472. }
  473. func (c *controller) activeCallback() {
  474. ds := c.getStore(datastore.GlobalScope)
  475. if ds != nil && !ds.Active() {
  476. ds.RestartWatch()
  477. }
  478. }
  479. func (c *controller) hostJoinCallback(nodes []net.IP) {
  480. c.processNodeDiscovery(nodes, true)
  481. }
  482. func (c *controller) hostLeaveCallback(nodes []net.IP) {
  483. c.processNodeDiscovery(nodes, false)
  484. }
  485. func (c *controller) processNodeDiscovery(nodes []net.IP, add bool) {
  486. c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool {
  487. c.pushNodeDiscovery(driver, capability, nodes, add)
  488. return false
  489. })
  490. }
  491. func (c *controller) pushNodeDiscovery(d driverapi.Driver, cap driverapi.Capability, nodes []net.IP, add bool) {
  492. var self net.IP
  493. if c.cfg != nil {
  494. addr := strings.Split(c.cfg.Cluster.Address, ":")
  495. self = net.ParseIP(addr[0])
  496. // if external kvstore is not configured, try swarm-mode config
  497. if self == nil {
  498. if agent := c.getAgent(); agent != nil {
  499. self = net.ParseIP(agent.advertiseAddr)
  500. }
  501. }
  502. }
  503. if d == nil || cap.DataScope != datastore.GlobalScope || nodes == nil {
  504. return
  505. }
  506. for _, node := range nodes {
  507. nodeData := discoverapi.NodeDiscoveryData{Address: node.String(), Self: node.Equal(self)}
  508. var err error
  509. if add {
  510. err = d.DiscoverNew(discoverapi.NodeDiscovery, nodeData)
  511. } else {
  512. err = d.DiscoverDelete(discoverapi.NodeDiscovery, nodeData)
  513. }
  514. if err != nil {
  515. logrus.Debugf("discovery notification error: %v", err)
  516. }
  517. }
  518. }
  519. func (c *controller) Config() config.Config {
  520. c.Lock()
  521. defer c.Unlock()
  522. if c.cfg == nil {
  523. return config.Config{}
  524. }
  525. return *c.cfg
  526. }
  527. func (c *controller) isManager() bool {
  528. c.Lock()
  529. defer c.Unlock()
  530. if c.cfg == nil || c.cfg.Daemon.ClusterProvider == nil {
  531. return false
  532. }
  533. return c.cfg.Daemon.ClusterProvider.IsManager()
  534. }
  535. func (c *controller) isAgent() bool {
  536. c.Lock()
  537. defer c.Unlock()
  538. if c.cfg == nil || c.cfg.Daemon.ClusterProvider == nil {
  539. return false
  540. }
  541. return c.cfg.Daemon.ClusterProvider.IsAgent()
  542. }
  543. func (c *controller) isDistributedControl() bool {
  544. return !c.isManager() && !c.isAgent()
  545. }
  546. func (c *controller) GetPluginGetter() plugingetter.PluginGetter {
  547. return c.drvRegistry.GetPluginGetter()
  548. }
  549. func (c *controller) RegisterDriver(networkType string, driver driverapi.Driver, capability driverapi.Capability) error {
  550. c.Lock()
  551. hd := c.discovery
  552. c.Unlock()
  553. if hd != nil {
  554. c.pushNodeDiscovery(driver, capability, hd.Fetch(), true)
  555. }
  556. c.agentDriverNotify(driver)
  557. return nil
  558. }
  559. // NewNetwork creates a new network of the specified network type. The options
  560. // are network specific and modeled in a generic way.
  561. func (c *controller) NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error) {
  562. if id != "" {
  563. c.networkLocker.Lock(id)
  564. defer c.networkLocker.Unlock(id)
  565. if _, err := c.NetworkByID(id); err == nil {
  566. return nil, NetworkNameError(id)
  567. }
  568. }
  569. if !config.IsValidName(name) {
  570. return nil, ErrInvalidName(name)
  571. }
  572. if id == "" {
  573. id = stringid.GenerateRandomID()
  574. }
  575. defaultIpam := defaultIpamForNetworkType(networkType)
  576. // Construct the network object
  577. network := &network{
  578. name: name,
  579. networkType: networkType,
  580. generic: map[string]interface{}{netlabel.GenericData: make(map[string]string)},
  581. ipamType: defaultIpam,
  582. id: id,
  583. created: time.Now(),
  584. ctrlr: c,
  585. persist: true,
  586. drvOnce: &sync.Once{},
  587. }
  588. network.processOptions(options...)
  589. _, cap, err := network.resolveDriver(networkType, true)
  590. if err != nil {
  591. return nil, err
  592. }
  593. if network.ingress && cap.DataScope != datastore.GlobalScope {
  594. return nil, types.ForbiddenErrorf("Ingress network can only be global scope network")
  595. }
  596. if cap.DataScope == datastore.GlobalScope && !c.isDistributedControl() && !network.dynamic {
  597. if c.isManager() {
  598. // For non-distributed controlled environment, globalscoped non-dynamic networks are redirected to Manager
  599. return nil, ManagerRedirectError(name)
  600. }
  601. return nil, types.ForbiddenErrorf("Cannot create a multi-host network from a worker node. Please create the network from a manager node.")
  602. }
  603. // Make sure we have a driver available for this network type
  604. // before we allocate anything.
  605. if _, err := network.driver(true); err != nil {
  606. return nil, err
  607. }
  608. err = network.ipamAllocate()
  609. if err != nil {
  610. return nil, err
  611. }
  612. defer func() {
  613. if err != nil {
  614. network.ipamRelease()
  615. }
  616. }()
  617. err = c.addNetwork(network)
  618. if err != nil {
  619. return nil, err
  620. }
  621. defer func() {
  622. if err != nil {
  623. if e := network.deleteNetwork(); e != nil {
  624. logrus.Warnf("couldn't roll back driver network on network %s creation failure: %v", network.name, err)
  625. }
  626. }
  627. }()
  628. // First store the endpoint count, then the network. To avoid to
  629. // end up with a datastore containing a network and not an epCnt,
  630. // in case of an ungraceful shutdown during this function call.
  631. epCnt := &endpointCnt{n: network}
  632. if err = c.updateToStore(epCnt); err != nil {
  633. return nil, err
  634. }
  635. defer func() {
  636. if err != nil {
  637. if e := c.deleteFromStore(epCnt); e != nil {
  638. logrus.Warnf("could not rollback from store, epCnt %v on failure (%v): %v", epCnt, err, e)
  639. }
  640. }
  641. }()
  642. network.epCnt = epCnt
  643. if err = c.updateToStore(network); err != nil {
  644. return nil, err
  645. }
  646. joinCluster(network)
  647. if !c.isDistributedControl() {
  648. c.Lock()
  649. arrangeIngressFilterRule()
  650. c.Unlock()
  651. }
  652. return network, nil
  653. }
  654. var joinCluster NetworkWalker = func(nw Network) bool {
  655. n := nw.(*network)
  656. if err := n.joinCluster(); err != nil {
  657. logrus.Errorf("Failed to join network %s (%s) into agent cluster: %v", n.Name(), n.ID(), err)
  658. }
  659. n.addDriverWatches()
  660. return false
  661. }
  662. func (c *controller) reservePools() {
  663. networks, err := c.getNetworksForScope(datastore.LocalScope)
  664. if err != nil {
  665. logrus.Warnf("Could not retrieve networks from local store during ipam allocation for existing networks: %v", err)
  666. return
  667. }
  668. for _, n := range networks {
  669. if !doReplayPoolReserve(n) {
  670. continue
  671. }
  672. // Construct pseudo configs for the auto IP case
  673. autoIPv4 := (len(n.ipamV4Config) == 0 || (len(n.ipamV4Config) == 1 && n.ipamV4Config[0].PreferredPool == "")) && len(n.ipamV4Info) > 0
  674. autoIPv6 := (len(n.ipamV6Config) == 0 || (len(n.ipamV6Config) == 1 && n.ipamV6Config[0].PreferredPool == "")) && len(n.ipamV6Info) > 0
  675. if autoIPv4 {
  676. n.ipamV4Config = []*IpamConf{{PreferredPool: n.ipamV4Info[0].Pool.String()}}
  677. }
  678. if n.enableIPv6 && autoIPv6 {
  679. n.ipamV6Config = []*IpamConf{{PreferredPool: n.ipamV6Info[0].Pool.String()}}
  680. }
  681. // Account current network gateways
  682. for i, c := range n.ipamV4Config {
  683. if c.Gateway == "" && n.ipamV4Info[i].Gateway != nil {
  684. c.Gateway = n.ipamV4Info[i].Gateway.IP.String()
  685. }
  686. }
  687. if n.enableIPv6 {
  688. for i, c := range n.ipamV6Config {
  689. if c.Gateway == "" && n.ipamV6Info[i].Gateway != nil {
  690. c.Gateway = n.ipamV6Info[i].Gateway.IP.String()
  691. }
  692. }
  693. }
  694. // Reserve pools
  695. if err := n.ipamAllocate(); err != nil {
  696. logrus.Warnf("Failed to allocate ipam pool(s) for network %q (%s): %v", n.Name(), n.ID(), err)
  697. }
  698. // Reserve existing endpoints' addresses
  699. ipam, _, err := n.getController().getIPAMDriver(n.ipamType)
  700. if err != nil {
  701. logrus.Warnf("Failed to retrieve ipam driver for network %q (%s) during address reservation", n.Name(), n.ID())
  702. continue
  703. }
  704. epl, err := n.getEndpointsFromStore()
  705. if err != nil {
  706. logrus.Warnf("Failed to retrieve list of current endpoints on network %q (%s)", n.Name(), n.ID())
  707. continue
  708. }
  709. for _, ep := range epl {
  710. if err := ep.assignAddress(ipam, true, ep.Iface().AddressIPv6() != nil); err != nil {
  711. logrus.Warnf("Failed to reserve current address for endpoint %q (%s) on network %q (%s)",
  712. ep.Name(), ep.ID(), n.Name(), n.ID())
  713. }
  714. }
  715. }
  716. }
  717. func doReplayPoolReserve(n *network) bool {
  718. _, caps, err := n.getController().getIPAMDriver(n.ipamType)
  719. if err != nil {
  720. logrus.Warnf("Failed to retrieve ipam driver for network %q (%s): %v", n.Name(), n.ID(), err)
  721. return false
  722. }
  723. return caps.RequiresRequestReplay
  724. }
  725. func (c *controller) addNetwork(n *network) error {
  726. d, err := n.driver(true)
  727. if err != nil {
  728. return err
  729. }
  730. // Create the network
  731. if err := d.CreateNetwork(n.id, n.generic, n, n.getIPData(4), n.getIPData(6)); err != nil {
  732. return err
  733. }
  734. n.startResolver()
  735. return nil
  736. }
  737. func (c *controller) Networks() []Network {
  738. var list []Network
  739. networks, err := c.getNetworksFromStore()
  740. if err != nil {
  741. logrus.Error(err)
  742. }
  743. for _, n := range networks {
  744. if n.inDelete {
  745. continue
  746. }
  747. list = append(list, n)
  748. }
  749. return list
  750. }
  751. func (c *controller) WalkNetworks(walker NetworkWalker) {
  752. for _, n := range c.Networks() {
  753. if walker(n) {
  754. return
  755. }
  756. }
  757. }
  758. func (c *controller) NetworkByName(name string) (Network, error) {
  759. if name == "" {
  760. return nil, ErrInvalidName(name)
  761. }
  762. var n Network
  763. s := func(current Network) bool {
  764. if current.Name() == name {
  765. n = current
  766. return true
  767. }
  768. return false
  769. }
  770. c.WalkNetworks(s)
  771. if n == nil {
  772. return nil, ErrNoSuchNetwork(name)
  773. }
  774. return n, nil
  775. }
  776. func (c *controller) NetworkByID(id string) (Network, error) {
  777. if id == "" {
  778. return nil, ErrInvalidID(id)
  779. }
  780. n, err := c.getNetworkFromStore(id)
  781. if err != nil {
  782. return nil, ErrNoSuchNetwork(id)
  783. }
  784. return n, nil
  785. }
  786. // NewSandbox creates a new sandbox for the passed container id
  787. func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (sBox Sandbox, err error) {
  788. if containerID == "" {
  789. return nil, types.BadRequestErrorf("invalid container ID")
  790. }
  791. var sb *sandbox
  792. c.Lock()
  793. for _, s := range c.sandboxes {
  794. if s.containerID == containerID {
  795. // If not a stub, then we already have a complete sandbox.
  796. if !s.isStub {
  797. sbID := s.ID()
  798. c.Unlock()
  799. return nil, types.ForbiddenErrorf("container %s is already present in sandbox %s", containerID, sbID)
  800. }
  801. // We already have a stub sandbox from the
  802. // store. Make use of it so that we don't lose
  803. // the endpoints from store but reset the
  804. // isStub flag.
  805. sb = s
  806. sb.isStub = false
  807. break
  808. }
  809. }
  810. c.Unlock()
  811. // Create sandbox and process options first. Key generation depends on an option
  812. if sb == nil {
  813. sb = &sandbox{
  814. id: stringid.GenerateRandomID(),
  815. containerID: containerID,
  816. endpoints: epHeap{},
  817. epPriority: map[string]int{},
  818. populatedEndpoints: map[string]struct{}{},
  819. config: containerConfig{},
  820. controller: c,
  821. extDNS: []extDNSEntry{},
  822. }
  823. }
  824. sBox = sb
  825. heap.Init(&sb.endpoints)
  826. sb.processOptions(options...)
  827. c.Lock()
  828. if sb.ingress && c.ingressSandbox != nil {
  829. c.Unlock()
  830. return nil, types.ForbiddenErrorf("ingress sandbox already present")
  831. }
  832. if sb.ingress {
  833. c.ingressSandbox = sb
  834. sb.id = "ingress_sbox"
  835. }
  836. c.Unlock()
  837. defer func() {
  838. if err != nil {
  839. c.Lock()
  840. if sb.ingress {
  841. c.ingressSandbox = nil
  842. }
  843. c.Unlock()
  844. }
  845. }()
  846. if err = sb.setupResolutionFiles(); err != nil {
  847. return nil, err
  848. }
  849. if sb.config.useDefaultSandBox {
  850. c.sboxOnce.Do(func() {
  851. c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
  852. })
  853. if err != nil {
  854. c.sboxOnce = sync.Once{}
  855. return nil, fmt.Errorf("failed to create default sandbox: %v", err)
  856. }
  857. sb.osSbox = c.defOsSbox
  858. }
  859. if sb.osSbox == nil && !sb.config.useExternalKey {
  860. if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil {
  861. return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
  862. }
  863. }
  864. c.Lock()
  865. c.sandboxes[sb.id] = sb
  866. c.Unlock()
  867. defer func() {
  868. if err != nil {
  869. c.Lock()
  870. delete(c.sandboxes, sb.id)
  871. c.Unlock()
  872. }
  873. }()
  874. err = sb.storeUpdate()
  875. if err != nil {
  876. return nil, fmt.Errorf("failed to update the store state of sandbox: %v", err)
  877. }
  878. return sb, nil
  879. }
  880. func (c *controller) Sandboxes() []Sandbox {
  881. c.Lock()
  882. defer c.Unlock()
  883. list := make([]Sandbox, 0, len(c.sandboxes))
  884. for _, s := range c.sandboxes {
  885. // Hide stub sandboxes from libnetwork users
  886. if s.isStub {
  887. continue
  888. }
  889. list = append(list, s)
  890. }
  891. return list
  892. }
  893. func (c *controller) WalkSandboxes(walker SandboxWalker) {
  894. for _, sb := range c.Sandboxes() {
  895. if walker(sb) {
  896. return
  897. }
  898. }
  899. }
  900. func (c *controller) SandboxByID(id string) (Sandbox, error) {
  901. if id == "" {
  902. return nil, ErrInvalidID(id)
  903. }
  904. c.Lock()
  905. s, ok := c.sandboxes[id]
  906. c.Unlock()
  907. if !ok {
  908. return nil, types.NotFoundErrorf("sandbox %s not found", id)
  909. }
  910. return s, nil
  911. }
  912. // SandboxDestroy destroys a sandbox given a container ID
  913. func (c *controller) SandboxDestroy(id string) error {
  914. var sb *sandbox
  915. c.Lock()
  916. for _, s := range c.sandboxes {
  917. if s.containerID == id {
  918. sb = s
  919. break
  920. }
  921. }
  922. c.Unlock()
  923. // It is not an error if sandbox is not available
  924. if sb == nil {
  925. return nil
  926. }
  927. return sb.Delete()
  928. }
  929. // SandboxContainerWalker returns a Sandbox Walker function which looks for an existing Sandbox with the passed containerID
  930. func SandboxContainerWalker(out *Sandbox, containerID string) SandboxWalker {
  931. return func(sb Sandbox) bool {
  932. if sb.ContainerID() == containerID {
  933. *out = sb
  934. return true
  935. }
  936. return false
  937. }
  938. }
  939. // SandboxKeyWalker returns a Sandbox Walker function which looks for an existing Sandbox with the passed key
  940. func SandboxKeyWalker(out *Sandbox, key string) SandboxWalker {
  941. return func(sb Sandbox) bool {
  942. if sb.Key() == key {
  943. *out = sb
  944. return true
  945. }
  946. return false
  947. }
  948. }
  949. func (c *controller) loadDriver(networkType string) error {
  950. var err error
  951. if pg := c.GetPluginGetter(); pg != nil {
  952. _, err = pg.Get(networkType, driverapi.NetworkPluginEndpointType, plugingetter.Lookup)
  953. } else {
  954. _, err = plugins.Get(networkType, driverapi.NetworkPluginEndpointType)
  955. }
  956. if err != nil {
  957. if err == plugins.ErrNotFound {
  958. return types.NotFoundErrorf(err.Error())
  959. }
  960. return err
  961. }
  962. return nil
  963. }
  964. func (c *controller) loadIPAMDriver(name string) error {
  965. var err error
  966. if pg := c.GetPluginGetter(); pg != nil {
  967. _, err = pg.Get(name, ipamapi.PluginEndpointType, plugingetter.Lookup)
  968. } else {
  969. _, err = plugins.Get(name, ipamapi.PluginEndpointType)
  970. }
  971. if err != nil {
  972. if err == plugins.ErrNotFound {
  973. return types.NotFoundErrorf(err.Error())
  974. }
  975. return err
  976. }
  977. return nil
  978. }
  979. func (c *controller) getIPAMDriver(name string) (ipamapi.Ipam, *ipamapi.Capability, error) {
  980. id, cap := c.drvRegistry.IPAM(name)
  981. if id == nil {
  982. // Might be a plugin name. Try loading it
  983. if err := c.loadIPAMDriver(name); err != nil {
  984. return nil, nil, err
  985. }
  986. // Now that we resolved the plugin, try again looking up the registry
  987. id, cap = c.drvRegistry.IPAM(name)
  988. if id == nil {
  989. return nil, nil, types.BadRequestErrorf("invalid ipam driver: %q", name)
  990. }
  991. }
  992. return id, cap, nil
  993. }
  994. func (c *controller) Stop() {
  995. c.closeStores()
  996. c.stopExternalKeyListener()
  997. osl.GC()
  998. }