endpoint.go 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206
  1. package libnetwork
  2. import (
  3. "container/heap"
  4. "encoding/json"
  5. "fmt"
  6. "net"
  7. "strings"
  8. "sync"
  9. "github.com/docker/libnetwork/datastore"
  10. "github.com/docker/libnetwork/ipamapi"
  11. "github.com/docker/libnetwork/netlabel"
  12. "github.com/docker/libnetwork/options"
  13. "github.com/docker/libnetwork/types"
  14. "github.com/sirupsen/logrus"
  15. )
  16. // Endpoint represents a logical connection between a network and a sandbox.
  17. type Endpoint interface {
  18. // A system generated id for this endpoint.
  19. ID() string
  20. // Name returns the name of this endpoint.
  21. Name() string
  22. // Network returns the name of the network to which this endpoint is attached.
  23. Network() string
  24. // Join joins the sandbox to the endpoint and populates into the sandbox
  25. // the network resources allocated for the endpoint.
  26. Join(sandbox Sandbox, options ...EndpointOption) error
  27. // Leave detaches the network resources populated in the sandbox.
  28. Leave(sandbox Sandbox, options ...EndpointOption) error
  29. // Return certain operational data belonging to this endpoint
  30. Info() EndpointInfo
  31. // DriverInfo returns a collection of driver operational data related to this endpoint retrieved from the driver
  32. DriverInfo() (map[string]interface{}, error)
  33. // Delete and detaches this endpoint from the network.
  34. Delete(force bool) error
  35. }
  36. // EndpointOption is an option setter function type used to pass various options to Network
  37. // and Endpoint interfaces methods. The various setter functions of type EndpointOption are
  38. // provided by libnetwork, they look like <Create|Join|Leave>Option[...](...)
  39. type EndpointOption func(ep *endpoint)
  40. type endpoint struct {
  41. name string
  42. id string
  43. network *network
  44. iface *endpointInterface
  45. joinInfo *endpointJoinInfo
  46. sandboxID string
  47. locator string
  48. exposedPorts []types.TransportPort
  49. anonymous bool
  50. disableResolution bool
  51. generic map[string]interface{}
  52. joinLeaveDone chan struct{}
  53. prefAddress net.IP
  54. prefAddressV6 net.IP
  55. ipamOptions map[string]string
  56. aliases map[string]string
  57. myAliases []string
  58. svcID string
  59. svcName string
  60. virtualIP net.IP
  61. svcAliases []string
  62. ingressPorts []*PortConfig
  63. dbIndex uint64
  64. dbExists bool
  65. serviceEnabled bool
  66. loadBalancer bool
  67. sync.Mutex
  68. }
  69. func (ep *endpoint) MarshalJSON() ([]byte, error) {
  70. ep.Lock()
  71. defer ep.Unlock()
  72. epMap := make(map[string]interface{})
  73. epMap["name"] = ep.name
  74. epMap["id"] = ep.id
  75. epMap["ep_iface"] = ep.iface
  76. epMap["joinInfo"] = ep.joinInfo
  77. epMap["exposed_ports"] = ep.exposedPorts
  78. if ep.generic != nil {
  79. epMap["generic"] = ep.generic
  80. }
  81. epMap["sandbox"] = ep.sandboxID
  82. epMap["locator"] = ep.locator
  83. epMap["anonymous"] = ep.anonymous
  84. epMap["disableResolution"] = ep.disableResolution
  85. epMap["myAliases"] = ep.myAliases
  86. epMap["svcName"] = ep.svcName
  87. epMap["svcID"] = ep.svcID
  88. epMap["virtualIP"] = ep.virtualIP.String()
  89. epMap["ingressPorts"] = ep.ingressPorts
  90. epMap["svcAliases"] = ep.svcAliases
  91. epMap["loadBalancer"] = ep.loadBalancer
  92. return json.Marshal(epMap)
  93. }
  94. func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
  95. ep.Lock()
  96. defer ep.Unlock()
  97. var epMap map[string]interface{}
  98. if err := json.Unmarshal(b, &epMap); err != nil {
  99. return err
  100. }
  101. ep.name = epMap["name"].(string)
  102. ep.id = epMap["id"].(string)
  103. ib, _ := json.Marshal(epMap["ep_iface"])
  104. json.Unmarshal(ib, &ep.iface)
  105. jb, _ := json.Marshal(epMap["joinInfo"])
  106. json.Unmarshal(jb, &ep.joinInfo)
  107. tb, _ := json.Marshal(epMap["exposed_ports"])
  108. var tPorts []types.TransportPort
  109. json.Unmarshal(tb, &tPorts)
  110. ep.exposedPorts = tPorts
  111. cb, _ := json.Marshal(epMap["sandbox"])
  112. json.Unmarshal(cb, &ep.sandboxID)
  113. if v, ok := epMap["generic"]; ok {
  114. ep.generic = v.(map[string]interface{})
  115. if opt, ok := ep.generic[netlabel.PortMap]; ok {
  116. pblist := []types.PortBinding{}
  117. for i := 0; i < len(opt.([]interface{})); i++ {
  118. pb := types.PortBinding{}
  119. tmp := opt.([]interface{})[i].(map[string]interface{})
  120. bytes, err := json.Marshal(tmp)
  121. if err != nil {
  122. logrus.Error(err)
  123. break
  124. }
  125. err = json.Unmarshal(bytes, &pb)
  126. if err != nil {
  127. logrus.Error(err)
  128. break
  129. }
  130. pblist = append(pblist, pb)
  131. }
  132. ep.generic[netlabel.PortMap] = pblist
  133. }
  134. if opt, ok := ep.generic[netlabel.ExposedPorts]; ok {
  135. tplist := []types.TransportPort{}
  136. for i := 0; i < len(opt.([]interface{})); i++ {
  137. tp := types.TransportPort{}
  138. tmp := opt.([]interface{})[i].(map[string]interface{})
  139. bytes, err := json.Marshal(tmp)
  140. if err != nil {
  141. logrus.Error(err)
  142. break
  143. }
  144. err = json.Unmarshal(bytes, &tp)
  145. if err != nil {
  146. logrus.Error(err)
  147. break
  148. }
  149. tplist = append(tplist, tp)
  150. }
  151. ep.generic[netlabel.ExposedPorts] = tplist
  152. }
  153. }
  154. if v, ok := epMap["anonymous"]; ok {
  155. ep.anonymous = v.(bool)
  156. }
  157. if v, ok := epMap["disableResolution"]; ok {
  158. ep.disableResolution = v.(bool)
  159. }
  160. if l, ok := epMap["locator"]; ok {
  161. ep.locator = l.(string)
  162. }
  163. if sn, ok := epMap["svcName"]; ok {
  164. ep.svcName = sn.(string)
  165. }
  166. if si, ok := epMap["svcID"]; ok {
  167. ep.svcID = si.(string)
  168. }
  169. if vip, ok := epMap["virtualIP"]; ok {
  170. ep.virtualIP = net.ParseIP(vip.(string))
  171. }
  172. if v, ok := epMap["loadBalancer"]; ok {
  173. ep.loadBalancer = v.(bool)
  174. }
  175. sal, _ := json.Marshal(epMap["svcAliases"])
  176. var svcAliases []string
  177. json.Unmarshal(sal, &svcAliases)
  178. ep.svcAliases = svcAliases
  179. pc, _ := json.Marshal(epMap["ingressPorts"])
  180. var ingressPorts []*PortConfig
  181. json.Unmarshal(pc, &ingressPorts)
  182. ep.ingressPorts = ingressPorts
  183. ma, _ := json.Marshal(epMap["myAliases"])
  184. var myAliases []string
  185. json.Unmarshal(ma, &myAliases)
  186. ep.myAliases = myAliases
  187. return nil
  188. }
  189. func (ep *endpoint) New() datastore.KVObject {
  190. return &endpoint{network: ep.getNetwork()}
  191. }
  192. func (ep *endpoint) CopyTo(o datastore.KVObject) error {
  193. ep.Lock()
  194. defer ep.Unlock()
  195. dstEp := o.(*endpoint)
  196. dstEp.name = ep.name
  197. dstEp.id = ep.id
  198. dstEp.sandboxID = ep.sandboxID
  199. dstEp.locator = ep.locator
  200. dstEp.dbIndex = ep.dbIndex
  201. dstEp.dbExists = ep.dbExists
  202. dstEp.anonymous = ep.anonymous
  203. dstEp.disableResolution = ep.disableResolution
  204. dstEp.svcName = ep.svcName
  205. dstEp.svcID = ep.svcID
  206. dstEp.virtualIP = ep.virtualIP
  207. dstEp.loadBalancer = ep.loadBalancer
  208. dstEp.svcAliases = make([]string, len(ep.svcAliases))
  209. copy(dstEp.svcAliases, ep.svcAliases)
  210. dstEp.ingressPorts = make([]*PortConfig, len(ep.ingressPorts))
  211. copy(dstEp.ingressPorts, ep.ingressPorts)
  212. if ep.iface != nil {
  213. dstEp.iface = &endpointInterface{}
  214. ep.iface.CopyTo(dstEp.iface)
  215. }
  216. if ep.joinInfo != nil {
  217. dstEp.joinInfo = &endpointJoinInfo{}
  218. ep.joinInfo.CopyTo(dstEp.joinInfo)
  219. }
  220. dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts))
  221. copy(dstEp.exposedPorts, ep.exposedPorts)
  222. dstEp.myAliases = make([]string, len(ep.myAliases))
  223. copy(dstEp.myAliases, ep.myAliases)
  224. dstEp.generic = options.Generic{}
  225. for k, v := range ep.generic {
  226. dstEp.generic[k] = v
  227. }
  228. return nil
  229. }
  230. func (ep *endpoint) ID() string {
  231. ep.Lock()
  232. defer ep.Unlock()
  233. return ep.id
  234. }
  235. func (ep *endpoint) Name() string {
  236. ep.Lock()
  237. defer ep.Unlock()
  238. return ep.name
  239. }
  240. func (ep *endpoint) MyAliases() []string {
  241. ep.Lock()
  242. defer ep.Unlock()
  243. return ep.myAliases
  244. }
  245. func (ep *endpoint) Network() string {
  246. if ep.network == nil {
  247. return ""
  248. }
  249. return ep.network.name
  250. }
  251. func (ep *endpoint) isAnonymous() bool {
  252. ep.Lock()
  253. defer ep.Unlock()
  254. return ep.anonymous
  255. }
  256. // enableService sets ep's serviceEnabled to the passed value if it's not in the
  257. // current state and returns true; false otherwise.
  258. func (ep *endpoint) enableService(state bool) bool {
  259. ep.Lock()
  260. defer ep.Unlock()
  261. if ep.serviceEnabled != state {
  262. ep.serviceEnabled = state
  263. return true
  264. }
  265. return false
  266. }
  267. func (ep *endpoint) needResolver() bool {
  268. ep.Lock()
  269. defer ep.Unlock()
  270. return !ep.disableResolution
  271. }
  272. // endpoint Key structure : endpoint/network-id/endpoint-id
  273. func (ep *endpoint) Key() []string {
  274. if ep.network == nil {
  275. return nil
  276. }
  277. return []string{datastore.EndpointKeyPrefix, ep.network.id, ep.id}
  278. }
  279. func (ep *endpoint) KeyPrefix() []string {
  280. if ep.network == nil {
  281. return nil
  282. }
  283. return []string{datastore.EndpointKeyPrefix, ep.network.id}
  284. }
  285. func (ep *endpoint) networkIDFromKey(key string) (string, error) {
  286. // endpoint Key structure : docker/libnetwork/endpoint/${network-id}/${endpoint-id}
  287. // it's an invalid key if the key doesn't have all the 5 key elements above
  288. keyElements := strings.Split(key, "/")
  289. if !strings.HasPrefix(key, datastore.Key(datastore.EndpointKeyPrefix)) || len(keyElements) < 5 {
  290. return "", fmt.Errorf("invalid endpoint key : %v", key)
  291. }
  292. // network-id is placed at index=3. pls refer to endpoint.Key() method
  293. return strings.Split(key, "/")[3], nil
  294. }
  295. func (ep *endpoint) Value() []byte {
  296. b, err := json.Marshal(ep)
  297. if err != nil {
  298. return nil
  299. }
  300. return b
  301. }
  302. func (ep *endpoint) SetValue(value []byte) error {
  303. return json.Unmarshal(value, ep)
  304. }
  305. func (ep *endpoint) Index() uint64 {
  306. ep.Lock()
  307. defer ep.Unlock()
  308. return ep.dbIndex
  309. }
  310. func (ep *endpoint) SetIndex(index uint64) {
  311. ep.Lock()
  312. defer ep.Unlock()
  313. ep.dbIndex = index
  314. ep.dbExists = true
  315. }
  316. func (ep *endpoint) Exists() bool {
  317. ep.Lock()
  318. defer ep.Unlock()
  319. return ep.dbExists
  320. }
  321. func (ep *endpoint) Skip() bool {
  322. return ep.getNetwork().Skip()
  323. }
  324. func (ep *endpoint) processOptions(options ...EndpointOption) {
  325. ep.Lock()
  326. defer ep.Unlock()
  327. for _, opt := range options {
  328. if opt != nil {
  329. opt(ep)
  330. }
  331. }
  332. }
  333. func (ep *endpoint) getNetwork() *network {
  334. ep.Lock()
  335. defer ep.Unlock()
  336. return ep.network
  337. }
  338. func (ep *endpoint) getNetworkFromStore() (*network, error) {
  339. if ep.network == nil {
  340. return nil, fmt.Errorf("invalid network object in endpoint %s", ep.Name())
  341. }
  342. return ep.network.getController().getNetworkFromStore(ep.network.id)
  343. }
  344. func (ep *endpoint) Join(sbox Sandbox, options ...EndpointOption) error {
  345. if sbox == nil {
  346. return types.BadRequestErrorf("endpoint cannot be joined by nil container")
  347. }
  348. sb, ok := sbox.(*sandbox)
  349. if !ok {
  350. return types.BadRequestErrorf("not a valid Sandbox interface")
  351. }
  352. sb.joinLeaveStart()
  353. defer sb.joinLeaveEnd()
  354. return ep.sbJoin(sb, options...)
  355. }
  356. func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) (err error) {
  357. n, err := ep.getNetworkFromStore()
  358. if err != nil {
  359. return fmt.Errorf("failed to get network from store during join: %v", err)
  360. }
  361. ep, err = n.getEndpointFromStore(ep.ID())
  362. if err != nil {
  363. return fmt.Errorf("failed to get endpoint from store during join: %v", err)
  364. }
  365. ep.Lock()
  366. if ep.sandboxID != "" {
  367. ep.Unlock()
  368. return types.ForbiddenErrorf("another container is attached to the same network endpoint")
  369. }
  370. ep.network = n
  371. ep.sandboxID = sb.ID()
  372. ep.joinInfo = &endpointJoinInfo{}
  373. epid := ep.id
  374. ep.Unlock()
  375. defer func() {
  376. if err != nil {
  377. ep.Lock()
  378. ep.sandboxID = ""
  379. ep.Unlock()
  380. }
  381. }()
  382. nid := n.ID()
  383. ep.processOptions(options...)
  384. d, err := n.driver(true)
  385. if err != nil {
  386. return fmt.Errorf("failed to get driver during join: %v", err)
  387. }
  388. err = d.Join(nid, epid, sb.Key(), ep, sb.Labels())
  389. if err != nil {
  390. return err
  391. }
  392. defer func() {
  393. if err != nil {
  394. if e := d.Leave(nid, epid); e != nil {
  395. logrus.Warnf("driver leave failed while rolling back join: %v", e)
  396. }
  397. }
  398. }()
  399. // Watch for service records
  400. if !n.getController().isAgent() {
  401. n.getController().watchSvcRecord(ep)
  402. }
  403. if doUpdateHostsFile(n, sb) {
  404. address := ""
  405. if ip := ep.getFirstInterfaceAddress(); ip != nil {
  406. address = ip.String()
  407. }
  408. if err = sb.updateHostsFile(address); err != nil {
  409. return err
  410. }
  411. }
  412. if err = sb.updateDNS(n.enableIPv6); err != nil {
  413. return err
  414. }
  415. // Current endpoint providing external connectivity for the sandbox
  416. extEp := sb.getGatewayEndpoint()
  417. sb.Lock()
  418. heap.Push(&sb.endpoints, ep)
  419. sb.Unlock()
  420. defer func() {
  421. if err != nil {
  422. sb.removeEndpoint(ep)
  423. }
  424. }()
  425. if err = sb.populateNetworkResources(ep); err != nil {
  426. return err
  427. }
  428. if err = n.getController().updateToStore(ep); err != nil {
  429. return err
  430. }
  431. if err = ep.addDriverInfoToCluster(); err != nil {
  432. return err
  433. }
  434. defer func() {
  435. if err != nil {
  436. if e := ep.deleteDriverInfoFromCluster(); e != nil {
  437. logrus.Errorf("Could not delete endpoint state for endpoint %s from cluster on join failure: %v", ep.Name(), e)
  438. }
  439. }
  440. }()
  441. if sb.needDefaultGW() && sb.getEndpointInGWNetwork() == nil {
  442. return sb.setupDefaultGW()
  443. }
  444. moveExtConn := sb.getGatewayEndpoint() != extEp
  445. if moveExtConn {
  446. if extEp != nil {
  447. logrus.Debugf("Revoking external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID())
  448. extN, err := extEp.getNetworkFromStore()
  449. if err != nil {
  450. return fmt.Errorf("failed to get network from store for revoking external connectivity during join: %v", err)
  451. }
  452. extD, err := extN.driver(true)
  453. if err != nil {
  454. return fmt.Errorf("failed to get driver for revoking external connectivity during join: %v", err)
  455. }
  456. if err = extD.RevokeExternalConnectivity(extEp.network.ID(), extEp.ID()); err != nil {
  457. return types.InternalErrorf(
  458. "driver failed revoking external connectivity on endpoint %s (%s): %v",
  459. extEp.Name(), extEp.ID(), err)
  460. }
  461. defer func() {
  462. if err != nil {
  463. if e := extD.ProgramExternalConnectivity(extEp.network.ID(), extEp.ID(), sb.Labels()); e != nil {
  464. logrus.Warnf("Failed to roll-back external connectivity on endpoint %s (%s): %v",
  465. extEp.Name(), extEp.ID(), e)
  466. }
  467. }
  468. }()
  469. }
  470. if !n.internal {
  471. logrus.Debugf("Programming external connectivity on endpoint %s (%s)", ep.Name(), ep.ID())
  472. if err = d.ProgramExternalConnectivity(n.ID(), ep.ID(), sb.Labels()); err != nil {
  473. return types.InternalErrorf(
  474. "driver failed programming external connectivity on endpoint %s (%s): %v",
  475. ep.Name(), ep.ID(), err)
  476. }
  477. }
  478. }
  479. if !sb.needDefaultGW() {
  480. if e := sb.clearDefaultGW(); e != nil {
  481. logrus.Warnf("Failure while disconnecting sandbox %s (%s) from gateway network: %v",
  482. sb.ID(), sb.ContainerID(), e)
  483. }
  484. }
  485. return nil
  486. }
  487. func doUpdateHostsFile(n *network, sb *sandbox) bool {
  488. return !n.ingress && n.Name() != libnGWNetwork
  489. }
  490. func (ep *endpoint) rename(name string) error {
  491. var (
  492. err error
  493. netWatch *netWatch
  494. ok bool
  495. )
  496. n := ep.getNetwork()
  497. if n == nil {
  498. return fmt.Errorf("network not connected for ep %q", ep.name)
  499. }
  500. c := n.getController()
  501. sb, ok := ep.getSandbox()
  502. if !ok {
  503. logrus.Warnf("rename for %s aborted, sandbox %s is not anymore present", ep.ID(), ep.sandboxID)
  504. return nil
  505. }
  506. if c.isAgent() {
  507. if err = ep.deleteServiceInfoFromCluster(sb, "rename"); err != nil {
  508. return types.InternalErrorf("Could not delete service state for endpoint %s from cluster on rename: %v", ep.Name(), err)
  509. }
  510. } else {
  511. c.Lock()
  512. netWatch, ok = c.nmap[n.ID()]
  513. c.Unlock()
  514. if !ok {
  515. return fmt.Errorf("watch null for network %q", n.Name())
  516. }
  517. n.updateSvcRecord(ep, c.getLocalEps(netWatch), false)
  518. }
  519. oldName := ep.name
  520. oldAnonymous := ep.anonymous
  521. ep.name = name
  522. ep.anonymous = false
  523. if c.isAgent() {
  524. if err = ep.addServiceInfoToCluster(sb); err != nil {
  525. return types.InternalErrorf("Could not add service state for endpoint %s to cluster on rename: %v", ep.Name(), err)
  526. }
  527. defer func() {
  528. if err != nil {
  529. ep.deleteServiceInfoFromCluster(sb, "rename")
  530. ep.name = oldName
  531. ep.anonymous = oldAnonymous
  532. ep.addServiceInfoToCluster(sb)
  533. }
  534. }()
  535. } else {
  536. n.updateSvcRecord(ep, c.getLocalEps(netWatch), true)
  537. defer func() {
  538. if err != nil {
  539. n.updateSvcRecord(ep, c.getLocalEps(netWatch), false)
  540. ep.name = oldName
  541. ep.anonymous = oldAnonymous
  542. n.updateSvcRecord(ep, c.getLocalEps(netWatch), true)
  543. }
  544. }()
  545. }
  546. // Update the store with the updated name
  547. if err = c.updateToStore(ep); err != nil {
  548. return err
  549. }
  550. // After the name change do a dummy endpoint count update to
  551. // trigger the service record update in the peer nodes
  552. // Ignore the error because updateStore fail for EpCnt is a
  553. // benign error. Besides there is no meaningful recovery that
  554. // we can do. When the cluster recovers subsequent EpCnt update
  555. // will force the peers to get the correct EP name.
  556. n.getEpCnt().updateStore()
  557. return err
  558. }
  559. func (ep *endpoint) hasInterface(iName string) bool {
  560. ep.Lock()
  561. defer ep.Unlock()
  562. return ep.iface != nil && ep.iface.srcName == iName
  563. }
  564. func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error {
  565. if sbox == nil || sbox.ID() == "" || sbox.Key() == "" {
  566. return types.BadRequestErrorf("invalid Sandbox passed to endpoint leave: %v", sbox)
  567. }
  568. sb, ok := sbox.(*sandbox)
  569. if !ok {
  570. return types.BadRequestErrorf("not a valid Sandbox interface")
  571. }
  572. sb.joinLeaveStart()
  573. defer sb.joinLeaveEnd()
  574. return ep.sbLeave(sb, false, options...)
  575. }
  576. func (ep *endpoint) sbLeave(sb *sandbox, force bool, options ...EndpointOption) error {
  577. n, err := ep.getNetworkFromStore()
  578. if err != nil {
  579. return fmt.Errorf("failed to get network from store during leave: %v", err)
  580. }
  581. ep, err = n.getEndpointFromStore(ep.ID())
  582. if err != nil {
  583. return fmt.Errorf("failed to get endpoint from store during leave: %v", err)
  584. }
  585. ep.Lock()
  586. sid := ep.sandboxID
  587. ep.Unlock()
  588. if sid == "" {
  589. return types.ForbiddenErrorf("cannot leave endpoint with no attached sandbox")
  590. }
  591. if sid != sb.ID() {
  592. return types.ForbiddenErrorf("unexpected sandbox ID in leave request. Expected %s. Got %s", ep.sandboxID, sb.ID())
  593. }
  594. ep.processOptions(options...)
  595. d, err := n.driver(!force)
  596. if err != nil {
  597. return fmt.Errorf("failed to get driver during endpoint leave: %v", err)
  598. }
  599. ep.Lock()
  600. ep.sandboxID = ""
  601. ep.network = n
  602. ep.Unlock()
  603. // Current endpoint providing external connectivity to the sandbox
  604. extEp := sb.getGatewayEndpoint()
  605. moveExtConn := extEp != nil && (extEp.ID() == ep.ID())
  606. if d != nil {
  607. if moveExtConn {
  608. logrus.Debugf("Revoking external connectivity on endpoint %s (%s)", ep.Name(), ep.ID())
  609. if err := d.RevokeExternalConnectivity(n.id, ep.id); err != nil {
  610. logrus.Warnf("driver failed revoking external connectivity on endpoint %s (%s): %v",
  611. ep.Name(), ep.ID(), err)
  612. }
  613. }
  614. if err := d.Leave(n.id, ep.id); err != nil {
  615. if _, ok := err.(types.MaskableError); !ok {
  616. logrus.Warnf("driver error disconnecting container %s : %v", ep.name, err)
  617. }
  618. }
  619. }
  620. if err := sb.clearNetworkResources(ep); err != nil {
  621. logrus.Warnf("Could not cleanup network resources on container %s disconnect: %v", ep.name, err)
  622. }
  623. // Update the store about the sandbox detach only after we
  624. // have completed sb.clearNetworkresources above to avoid
  625. // spurious logs when cleaning up the sandbox when the daemon
  626. // ungracefully exits and restarts before completing sandbox
  627. // detach but after store has been updated.
  628. if err := n.getController().updateToStore(ep); err != nil {
  629. return err
  630. }
  631. if e := ep.deleteServiceInfoFromCluster(sb, "sbLeave"); e != nil {
  632. logrus.Errorf("Could not delete service state for endpoint %s from cluster: %v", ep.Name(), e)
  633. }
  634. if e := ep.deleteDriverInfoFromCluster(); e != nil {
  635. logrus.Errorf("Could not delete endpoint state for endpoint %s from cluster: %v", ep.Name(), e)
  636. }
  637. sb.deleteHostsEntries(n.getSvcRecords(ep))
  638. if !sb.inDelete && sb.needDefaultGW() && sb.getEndpointInGWNetwork() == nil {
  639. return sb.setupDefaultGW()
  640. }
  641. // New endpoint providing external connectivity for the sandbox
  642. extEp = sb.getGatewayEndpoint()
  643. if moveExtConn && extEp != nil {
  644. logrus.Debugf("Programming external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID())
  645. extN, err := extEp.getNetworkFromStore()
  646. if err != nil {
  647. return fmt.Errorf("failed to get network from store for programming external connectivity during leave: %v", err)
  648. }
  649. extD, err := extN.driver(true)
  650. if err != nil {
  651. return fmt.Errorf("failed to get driver for programming external connectivity during leave: %v", err)
  652. }
  653. if err := extD.ProgramExternalConnectivity(extEp.network.ID(), extEp.ID(), sb.Labels()); err != nil {
  654. logrus.Warnf("driver failed programming external connectivity on endpoint %s: (%s) %v",
  655. extEp.Name(), extEp.ID(), err)
  656. }
  657. }
  658. if !sb.needDefaultGW() {
  659. if err := sb.clearDefaultGW(); err != nil {
  660. logrus.Warnf("Failure while disconnecting sandbox %s (%s) from gateway network: %v",
  661. sb.ID(), sb.ContainerID(), err)
  662. }
  663. }
  664. return nil
  665. }
  666. func (ep *endpoint) Delete(force bool) error {
  667. var err error
  668. n, err := ep.getNetworkFromStore()
  669. if err != nil {
  670. return fmt.Errorf("failed to get network during Delete: %v", err)
  671. }
  672. ep, err = n.getEndpointFromStore(ep.ID())
  673. if err != nil {
  674. return fmt.Errorf("failed to get endpoint from store during Delete: %v", err)
  675. }
  676. ep.Lock()
  677. epid := ep.id
  678. name := ep.name
  679. sbid := ep.sandboxID
  680. ep.Unlock()
  681. sb, _ := n.getController().SandboxByID(sbid)
  682. if sb != nil && !force {
  683. return &ActiveContainerError{name: name, id: epid}
  684. }
  685. if sb != nil {
  686. if e := ep.sbLeave(sb.(*sandbox), force); e != nil {
  687. logrus.Warnf("failed to leave sandbox for endpoint %s : %v", name, e)
  688. }
  689. }
  690. if err = n.getController().deleteFromStore(ep); err != nil {
  691. return err
  692. }
  693. defer func() {
  694. if err != nil && !force {
  695. ep.dbExists = false
  696. if e := n.getController().updateToStore(ep); e != nil {
  697. logrus.Warnf("failed to recreate endpoint in store %s : %v", name, e)
  698. }
  699. }
  700. }()
  701. // unwatch for service records
  702. n.getController().unWatchSvcRecord(ep)
  703. if err = ep.deleteEndpoint(force); err != nil && !force {
  704. return err
  705. }
  706. ep.releaseAddress()
  707. if err := n.getEpCnt().DecEndpointCnt(); err != nil {
  708. logrus.Warnf("failed to decrement endpoint count for ep %s: %v", ep.ID(), err)
  709. }
  710. return nil
  711. }
  712. func (ep *endpoint) deleteEndpoint(force bool) error {
  713. ep.Lock()
  714. n := ep.network
  715. name := ep.name
  716. epid := ep.id
  717. ep.Unlock()
  718. driver, err := n.driver(!force)
  719. if err != nil {
  720. return fmt.Errorf("failed to delete endpoint: %v", err)
  721. }
  722. if driver == nil {
  723. return nil
  724. }
  725. if err := driver.DeleteEndpoint(n.id, epid); err != nil {
  726. if _, ok := err.(types.ForbiddenError); ok {
  727. return err
  728. }
  729. if _, ok := err.(types.MaskableError); !ok {
  730. logrus.Warnf("driver error deleting endpoint %s : %v", name, err)
  731. }
  732. }
  733. return nil
  734. }
  735. func (ep *endpoint) getSandbox() (*sandbox, bool) {
  736. c := ep.network.getController()
  737. ep.Lock()
  738. sid := ep.sandboxID
  739. ep.Unlock()
  740. c.Lock()
  741. ps, ok := c.sandboxes[sid]
  742. c.Unlock()
  743. return ps, ok
  744. }
  745. func (ep *endpoint) getFirstInterfaceAddress() net.IP {
  746. ep.Lock()
  747. defer ep.Unlock()
  748. if ep.iface.addr != nil {
  749. return ep.iface.addr.IP
  750. }
  751. return nil
  752. }
  753. // EndpointOptionGeneric function returns an option setter for a Generic option defined
  754. // in a Dictionary of Key-Value pair
  755. func EndpointOptionGeneric(generic map[string]interface{}) EndpointOption {
  756. return func(ep *endpoint) {
  757. for k, v := range generic {
  758. ep.generic[k] = v
  759. }
  760. }
  761. }
  762. var (
  763. linkLocalMask = net.CIDRMask(16, 32)
  764. linkLocalMaskIPv6 = net.CIDRMask(64, 128)
  765. )
  766. // CreateOptionIpam function returns an option setter for the ipam configuration for this endpoint
  767. func CreateOptionIpam(ipV4, ipV6 net.IP, llIPs []net.IP, ipamOptions map[string]string) EndpointOption {
  768. return func(ep *endpoint) {
  769. ep.prefAddress = ipV4
  770. ep.prefAddressV6 = ipV6
  771. if len(llIPs) != 0 {
  772. for _, ip := range llIPs {
  773. nw := &net.IPNet{IP: ip, Mask: linkLocalMask}
  774. if ip.To4() == nil {
  775. nw.Mask = linkLocalMaskIPv6
  776. }
  777. ep.iface.llAddrs = append(ep.iface.llAddrs, nw)
  778. }
  779. }
  780. ep.ipamOptions = ipamOptions
  781. }
  782. }
  783. // CreateOptionExposedPorts function returns an option setter for the container exposed
  784. // ports option to be passed to network.CreateEndpoint() method.
  785. func CreateOptionExposedPorts(exposedPorts []types.TransportPort) EndpointOption {
  786. return func(ep *endpoint) {
  787. // Defensive copy
  788. eps := make([]types.TransportPort, len(exposedPorts))
  789. copy(eps, exposedPorts)
  790. // Store endpoint label and in generic because driver needs it
  791. ep.exposedPorts = eps
  792. ep.generic[netlabel.ExposedPorts] = eps
  793. }
  794. }
  795. // CreateOptionPortMapping function returns an option setter for the mapping
  796. // ports option to be passed to network.CreateEndpoint() method.
  797. func CreateOptionPortMapping(portBindings []types.PortBinding) EndpointOption {
  798. return func(ep *endpoint) {
  799. // Store a copy of the bindings as generic data to pass to the driver
  800. pbs := make([]types.PortBinding, len(portBindings))
  801. copy(pbs, portBindings)
  802. ep.generic[netlabel.PortMap] = pbs
  803. }
  804. }
  805. // CreateOptionDNS function returns an option setter for dns entry option to
  806. // be passed to container Create method.
  807. func CreateOptionDNS(dns []string) EndpointOption {
  808. return func(ep *endpoint) {
  809. ep.generic[netlabel.DNSServers] = dns
  810. }
  811. }
  812. // CreateOptionAnonymous function returns an option setter for setting
  813. // this endpoint as anonymous
  814. func CreateOptionAnonymous() EndpointOption {
  815. return func(ep *endpoint) {
  816. ep.anonymous = true
  817. }
  818. }
  819. // CreateOptionDisableResolution function returns an option setter to indicate
  820. // this endpoint doesn't want embedded DNS server functionality
  821. func CreateOptionDisableResolution() EndpointOption {
  822. return func(ep *endpoint) {
  823. ep.disableResolution = true
  824. }
  825. }
  826. // CreateOptionAlias function returns an option setter for setting endpoint alias
  827. func CreateOptionAlias(name string, alias string) EndpointOption {
  828. return func(ep *endpoint) {
  829. if ep.aliases == nil {
  830. ep.aliases = make(map[string]string)
  831. }
  832. ep.aliases[alias] = name
  833. }
  834. }
  835. // CreateOptionService function returns an option setter for setting service binding configuration
  836. func CreateOptionService(name, id string, vip net.IP, ingressPorts []*PortConfig, aliases []string) EndpointOption {
  837. return func(ep *endpoint) {
  838. ep.svcName = name
  839. ep.svcID = id
  840. ep.virtualIP = vip
  841. ep.ingressPorts = ingressPorts
  842. ep.svcAliases = aliases
  843. }
  844. }
  845. // CreateOptionMyAlias function returns an option setter for setting endpoint's self alias
  846. func CreateOptionMyAlias(alias string) EndpointOption {
  847. return func(ep *endpoint) {
  848. ep.myAliases = append(ep.myAliases, alias)
  849. }
  850. }
  851. // CreateOptionLoadBalancer function returns an option setter for denoting the endpoint is a load balancer for a network
  852. func CreateOptionLoadBalancer() EndpointOption {
  853. return func(ep *endpoint) {
  854. ep.loadBalancer = true
  855. }
  856. }
  857. // JoinOptionPriority function returns an option setter for priority option to
  858. // be passed to the endpoint.Join() method.
  859. func JoinOptionPriority(ep Endpoint, prio int) EndpointOption {
  860. return func(ep *endpoint) {
  861. // ep lock already acquired
  862. c := ep.network.getController()
  863. c.Lock()
  864. sb, ok := c.sandboxes[ep.sandboxID]
  865. c.Unlock()
  866. if !ok {
  867. logrus.Errorf("Could not set endpoint priority value during Join to endpoint %s: No sandbox id present in endpoint", ep.id)
  868. return
  869. }
  870. sb.epPriority[ep.id] = prio
  871. }
  872. }
  873. func (ep *endpoint) DataScope() string {
  874. return ep.getNetwork().DataScope()
  875. }
  876. func (ep *endpoint) assignAddress(ipam ipamapi.Ipam, assignIPv4, assignIPv6 bool) error {
  877. var err error
  878. n := ep.getNetwork()
  879. if n.hasSpecialDriver() {
  880. return nil
  881. }
  882. logrus.Debugf("Assigning addresses for endpoint %s's interface on network %s", ep.Name(), n.Name())
  883. if assignIPv4 {
  884. if err = ep.assignAddressVersion(4, ipam); err != nil {
  885. return err
  886. }
  887. }
  888. if assignIPv6 {
  889. err = ep.assignAddressVersion(6, ipam)
  890. }
  891. return err
  892. }
  893. func (ep *endpoint) assignAddressVersion(ipVer int, ipam ipamapi.Ipam) error {
  894. var (
  895. poolID *string
  896. address **net.IPNet
  897. prefAdd net.IP
  898. progAdd net.IP
  899. )
  900. n := ep.getNetwork()
  901. switch ipVer {
  902. case 4:
  903. poolID = &ep.iface.v4PoolID
  904. address = &ep.iface.addr
  905. prefAdd = ep.prefAddress
  906. case 6:
  907. poolID = &ep.iface.v6PoolID
  908. address = &ep.iface.addrv6
  909. prefAdd = ep.prefAddressV6
  910. default:
  911. return types.InternalErrorf("incorrect ip version number passed: %d", ipVer)
  912. }
  913. ipInfo := n.getIPInfo(ipVer)
  914. // ipv6 address is not mandatory
  915. if len(ipInfo) == 0 && ipVer == 6 {
  916. return nil
  917. }
  918. // The address to program may be chosen by the user or by the network driver in one specific
  919. // case to support backward compatibility with `docker daemon --fixed-cidrv6` use case
  920. if prefAdd != nil {
  921. progAdd = prefAdd
  922. } else if *address != nil {
  923. progAdd = (*address).IP
  924. }
  925. for _, d := range ipInfo {
  926. if progAdd != nil && !d.Pool.Contains(progAdd) {
  927. continue
  928. }
  929. addr, _, err := ipam.RequestAddress(d.PoolID, progAdd, ep.ipamOptions)
  930. if err == nil {
  931. ep.Lock()
  932. *address = addr
  933. *poolID = d.PoolID
  934. ep.Unlock()
  935. return nil
  936. }
  937. if err != ipamapi.ErrNoAvailableIPs || progAdd != nil {
  938. return err
  939. }
  940. }
  941. if progAdd != nil {
  942. return types.BadRequestErrorf("Invalid address %s: It does not belong to any of this network's subnets", prefAdd)
  943. }
  944. return fmt.Errorf("no available IPv%d addresses on this network's address pools: %s (%s)", ipVer, n.Name(), n.ID())
  945. }
  946. func (ep *endpoint) releaseAddress() {
  947. n := ep.getNetwork()
  948. if n.hasSpecialDriver() {
  949. return
  950. }
  951. logrus.Debugf("Releasing addresses for endpoint %s's interface on network %s", ep.Name(), n.Name())
  952. ipam, _, err := n.getController().getIPAMDriver(n.ipamType)
  953. if err != nil {
  954. logrus.Warnf("Failed to retrieve ipam driver to release interface address on delete of endpoint %s (%s): %v", ep.Name(), ep.ID(), err)
  955. return
  956. }
  957. if ep.iface.addr != nil {
  958. if err := ipam.ReleaseAddress(ep.iface.v4PoolID, ep.iface.addr.IP); err != nil {
  959. logrus.Warnf("Failed to release ip address %s on delete of endpoint %s (%s): %v", ep.iface.addr.IP, ep.Name(), ep.ID(), err)
  960. }
  961. }
  962. if ep.iface.addrv6 != nil && ep.iface.addrv6.IP.IsGlobalUnicast() {
  963. if err := ipam.ReleaseAddress(ep.iface.v6PoolID, ep.iface.addrv6.IP); err != nil {
  964. logrus.Warnf("Failed to release ip address %s on delete of endpoint %s (%s): %v", ep.iface.addrv6.IP, ep.Name(), ep.ID(), err)
  965. }
  966. }
  967. }
  968. func (c *controller) cleanupLocalEndpoints() {
  969. // Get used endpoints
  970. eps := make(map[string]interface{})
  971. for _, sb := range c.sandboxes {
  972. for _, ep := range sb.endpoints {
  973. eps[ep.id] = true
  974. }
  975. }
  976. nl, err := c.getNetworksForScope(datastore.LocalScope)
  977. if err != nil {
  978. logrus.Warnf("Could not get list of networks during endpoint cleanup: %v", err)
  979. return
  980. }
  981. for _, n := range nl {
  982. if n.ConfigOnly() {
  983. continue
  984. }
  985. epl, err := n.getEndpointsFromStore()
  986. if err != nil {
  987. logrus.Warnf("Could not get list of endpoints in network %s during endpoint cleanup: %v", n.name, err)
  988. continue
  989. }
  990. for _, ep := range epl {
  991. if _, ok := eps[ep.id]; ok {
  992. continue
  993. }
  994. logrus.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id)
  995. if err := ep.Delete(true); err != nil {
  996. logrus.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)
  997. }
  998. }
  999. epl, err = n.getEndpointsFromStore()
  1000. if err != nil {
  1001. logrus.Warnf("Could not get list of endpoints in network %s for count update: %v", n.name, err)
  1002. continue
  1003. }
  1004. epCnt := n.getEpCnt().EndpointCnt()
  1005. if epCnt != uint64(len(epl)) {
  1006. logrus.Infof("Fixing inconsistent endpoint_cnt for network %s. Expected=%d, Actual=%d", n.name, len(epl), epCnt)
  1007. n.getEpCnt().setCnt(uint64(len(epl)))
  1008. }
  1009. }
  1010. }