sandbox.go 31 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237
  1. package libnetwork
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "net"
  6. "sort"
  7. "strings"
  8. "sync"
  9. "time"
  10. "github.com/docker/docker/libnetwork/etchosts"
  11. "github.com/docker/docker/libnetwork/netlabel"
  12. "github.com/docker/docker/libnetwork/osl"
  13. "github.com/docker/docker/libnetwork/types"
  14. "github.com/sirupsen/logrus"
  15. )
  16. // SandboxOption is an option setter function type used to pass various options to
  17. // NewNetContainer method. The various setter functions of type SandboxOption are
  18. // provided by libnetwork, they look like ContainerOptionXXXX(...)
  19. type SandboxOption func(sb *Sandbox)
  20. func (sb *Sandbox) processOptions(options ...SandboxOption) {
  21. for _, opt := range options {
  22. if opt != nil {
  23. opt(sb)
  24. }
  25. }
  26. }
  27. // Sandbox provides the control over the network container entity.
  28. // It is a one to one mapping with the container.
  29. type Sandbox struct {
  30. id string
  31. containerID string
  32. config containerConfig
  33. extDNS []extDNSEntry
  34. osSbox osl.Sandbox
  35. controller *Controller
  36. resolver *Resolver
  37. resolverOnce sync.Once
  38. endpoints []*Endpoint
  39. epPriority map[string]int
  40. populatedEndpoints map[string]struct{}
  41. joinLeaveDone chan struct{}
  42. dbIndex uint64
  43. dbExists bool
  44. isStub bool
  45. inDelete bool
  46. ingress bool
  47. ndotsSet bool
  48. oslTypes []osl.SandboxType // slice of properties of this sandbox
  49. loadBalancerNID string // NID that this SB is a load balancer for
  50. mu sync.Mutex
  51. // This mutex is used to serialize service related operation for an endpoint
  52. // The lock is here because the endpoint is saved into the store so is not unique
  53. service sync.Mutex
  54. }
  55. // These are the container configs used to customize container /etc/hosts file.
  56. type hostsPathConfig struct {
  57. hostName string
  58. domainName string
  59. hostsPath string
  60. originHostsPath string
  61. extraHosts []extraHost
  62. parentUpdates []parentUpdate
  63. }
  64. type parentUpdate struct {
  65. cid string
  66. name string
  67. ip string
  68. }
  69. type extraHost struct {
  70. name string
  71. IP string
  72. }
  73. // These are the container configs used to customize container /etc/resolv.conf file.
  74. type resolvConfPathConfig struct {
  75. resolvConfPath string
  76. originResolvConfPath string
  77. resolvConfHashFile string
  78. dnsList []string
  79. dnsSearchList []string
  80. dnsOptionsList []string
  81. }
  82. type containerConfig struct {
  83. hostsPathConfig
  84. resolvConfPathConfig
  85. generic map[string]interface{}
  86. useDefaultSandBox bool
  87. useExternalKey bool
  88. exposedPorts []types.TransportPort
  89. }
  90. const (
  91. resolverIPSandbox = "127.0.0.11"
  92. )
  93. // ID returns the ID of the sandbox.
  94. func (sb *Sandbox) ID() string {
  95. return sb.id
  96. }
  97. // ContainerID returns the container id associated to this sandbox.
  98. func (sb *Sandbox) ContainerID() string {
  99. return sb.containerID
  100. }
  101. // Key returns the sandbox's key.
  102. func (sb *Sandbox) Key() string {
  103. if sb.config.useDefaultSandBox {
  104. return osl.GenerateKey("default")
  105. }
  106. return osl.GenerateKey(sb.id)
  107. }
  108. // Labels returns the sandbox's labels.
  109. func (sb *Sandbox) Labels() map[string]interface{} {
  110. sb.mu.Lock()
  111. defer sb.mu.Unlock()
  112. opts := make(map[string]interface{}, len(sb.config.generic))
  113. for k, v := range sb.config.generic {
  114. opts[k] = v
  115. }
  116. return opts
  117. }
  118. // Statistics retrieves the interfaces' statistics for the sandbox.
  119. func (sb *Sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) {
  120. m := make(map[string]*types.InterfaceStatistics)
  121. sb.mu.Lock()
  122. osb := sb.osSbox
  123. sb.mu.Unlock()
  124. if osb == nil {
  125. return m, nil
  126. }
  127. var err error
  128. for _, i := range osb.Info().Interfaces() {
  129. if m[i.DstName()], err = i.Statistics(); err != nil {
  130. return m, err
  131. }
  132. }
  133. return m, nil
  134. }
  135. // Delete destroys this container after detaching it from all connected endpoints.
  136. func (sb *Sandbox) Delete() error {
  137. return sb.delete(false)
  138. }
  139. func (sb *Sandbox) delete(force bool) error {
  140. sb.mu.Lock()
  141. if sb.inDelete {
  142. sb.mu.Unlock()
  143. return types.ForbiddenErrorf("another sandbox delete in progress")
  144. }
  145. // Set the inDelete flag. This will ensure that we don't
  146. // update the store until we have completed all the endpoint
  147. // leaves and deletes. And when endpoint leaves and deletes
  148. // are completed then we can finally delete the sandbox object
  149. // altogether from the data store. If the daemon exits
  150. // ungracefully in the middle of a sandbox delete this way we
  151. // will have all the references to the endpoints in the
  152. // sandbox so that we can clean them up when we restart
  153. sb.inDelete = true
  154. sb.mu.Unlock()
  155. c := sb.controller
  156. // Detach from all endpoints
  157. retain := false
  158. for _, ep := range sb.Endpoints() {
  159. // gw network endpoint detach and removal are automatic
  160. if ep.endpointInGWNetwork() && !force {
  161. continue
  162. }
  163. // Retain the sanbdox if we can't obtain the network from store.
  164. if _, err := c.getNetworkFromStore(ep.getNetwork().ID()); err != nil {
  165. if c.isDistributedControl() {
  166. retain = true
  167. }
  168. logrus.Warnf("Failed getting network for ep %s during sandbox %s delete: %v", ep.ID(), sb.ID(), err)
  169. continue
  170. }
  171. if !force {
  172. if err := ep.Leave(sb); err != nil {
  173. logrus.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
  174. }
  175. }
  176. if err := ep.Delete(force); err != nil {
  177. logrus.Warnf("Failed deleting endpoint %s: %v\n", ep.ID(), err)
  178. }
  179. }
  180. if retain {
  181. sb.mu.Lock()
  182. sb.inDelete = false
  183. sb.mu.Unlock()
  184. return fmt.Errorf("could not cleanup all the endpoints in container %s / sandbox %s", sb.containerID, sb.id)
  185. }
  186. // Container is going away. Path cache in etchosts is most
  187. // likely not required any more. Drop it.
  188. etchosts.Drop(sb.config.hostsPath)
  189. if sb.resolver != nil {
  190. sb.resolver.Stop()
  191. }
  192. if sb.osSbox != nil && !sb.config.useDefaultSandBox {
  193. if err := sb.osSbox.Destroy(); err != nil {
  194. logrus.WithError(err).Warn("error destroying network sandbox")
  195. }
  196. }
  197. if err := sb.storeDelete(); err != nil {
  198. logrus.Warnf("Failed to delete sandbox %s from store: %v", sb.ID(), err)
  199. }
  200. c.mu.Lock()
  201. if sb.ingress {
  202. c.ingressSandbox = nil
  203. }
  204. delete(c.sandboxes, sb.ID())
  205. c.mu.Unlock()
  206. return nil
  207. }
  208. // Rename changes the name of all attached Endpoints.
  209. func (sb *Sandbox) Rename(name string) error {
  210. var err error
  211. for _, ep := range sb.Endpoints() {
  212. if ep.endpointInGWNetwork() {
  213. continue
  214. }
  215. oldName := ep.Name()
  216. lEp := ep
  217. if err = ep.rename(name); err != nil {
  218. break
  219. }
  220. defer func() {
  221. if err != nil {
  222. if err2 := lEp.rename(oldName); err2 != nil {
  223. logrus.WithField("old", oldName).WithField("origError", err).WithError(err2).Error("error renaming sandbox")
  224. }
  225. }
  226. }()
  227. }
  228. return err
  229. }
  230. // Refresh leaves all the endpoints, resets and re-applies the options,
  231. // re-joins all the endpoints without destroying the osl sandbox
  232. func (sb *Sandbox) Refresh(options ...SandboxOption) error {
  233. // Store connected endpoints
  234. epList := sb.Endpoints()
  235. // Detach from all endpoints
  236. for _, ep := range epList {
  237. if err := ep.Leave(sb); err != nil {
  238. logrus.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
  239. }
  240. }
  241. // Re-apply options
  242. sb.config = containerConfig{}
  243. sb.processOptions(options...)
  244. // Setup discovery files
  245. if err := sb.setupResolutionFiles(); err != nil {
  246. return err
  247. }
  248. // Re-connect to all endpoints
  249. for _, ep := range epList {
  250. if err := ep.Join(sb); err != nil {
  251. logrus.Warnf("Failed attach sandbox %s to endpoint %s: %v\n", sb.ID(), ep.ID(), err)
  252. }
  253. }
  254. return nil
  255. }
  256. func (sb *Sandbox) MarshalJSON() ([]byte, error) {
  257. sb.mu.Lock()
  258. defer sb.mu.Unlock()
  259. // We are just interested in the container ID. This can be expanded to include all of containerInfo if there is a need
  260. return json.Marshal(sb.id)
  261. }
  262. func (sb *Sandbox) UnmarshalJSON(b []byte) (err error) {
  263. sb.mu.Lock()
  264. defer sb.mu.Unlock()
  265. var id string
  266. if err := json.Unmarshal(b, &id); err != nil {
  267. return err
  268. }
  269. sb.id = id
  270. return nil
  271. }
  272. // Endpoints returns all the endpoints connected to the sandbox.
  273. func (sb *Sandbox) Endpoints() []*Endpoint {
  274. sb.mu.Lock()
  275. defer sb.mu.Unlock()
  276. eps := make([]*Endpoint, len(sb.endpoints))
  277. copy(eps, sb.endpoints)
  278. return eps
  279. }
  280. func (sb *Sandbox) addEndpoint(ep *Endpoint) {
  281. sb.mu.Lock()
  282. defer sb.mu.Unlock()
  283. l := len(sb.endpoints)
  284. i := sort.Search(l, func(j int) bool {
  285. return ep.Less(sb.endpoints[j])
  286. })
  287. sb.endpoints = append(sb.endpoints, nil)
  288. copy(sb.endpoints[i+1:], sb.endpoints[i:])
  289. sb.endpoints[i] = ep
  290. }
  291. func (sb *Sandbox) removeEndpoint(ep *Endpoint) {
  292. sb.mu.Lock()
  293. defer sb.mu.Unlock()
  294. sb.removeEndpointRaw(ep)
  295. }
  296. func (sb *Sandbox) removeEndpointRaw(ep *Endpoint) {
  297. for i, e := range sb.endpoints {
  298. if e == ep {
  299. sb.endpoints = append(sb.endpoints[:i], sb.endpoints[i+1:]...)
  300. return
  301. }
  302. }
  303. }
  304. func (sb *Sandbox) getEndpoint(id string) *Endpoint {
  305. sb.mu.Lock()
  306. defer sb.mu.Unlock()
  307. for _, ep := range sb.endpoints {
  308. if ep.id == id {
  309. return ep
  310. }
  311. }
  312. return nil
  313. }
  314. func (sb *Sandbox) updateGateway(ep *Endpoint) error {
  315. sb.mu.Lock()
  316. osSbox := sb.osSbox
  317. sb.mu.Unlock()
  318. if osSbox == nil {
  319. return nil
  320. }
  321. osSbox.UnsetGateway() //nolint:errcheck
  322. osSbox.UnsetGatewayIPv6() //nolint:errcheck
  323. if ep == nil {
  324. return nil
  325. }
  326. ep.mu.Lock()
  327. joinInfo := ep.joinInfo
  328. ep.mu.Unlock()
  329. if err := osSbox.SetGateway(joinInfo.gw); err != nil {
  330. return fmt.Errorf("failed to set gateway while updating gateway: %v", err)
  331. }
  332. if err := osSbox.SetGatewayIPv6(joinInfo.gw6); err != nil {
  333. return fmt.Errorf("failed to set IPv6 gateway while updating gateway: %v", err)
  334. }
  335. return nil
  336. }
  337. func (sb *Sandbox) HandleQueryResp(name string, ip net.IP) {
  338. for _, ep := range sb.Endpoints() {
  339. n := ep.getNetwork()
  340. n.HandleQueryResp(name, ip)
  341. }
  342. }
  343. func (sb *Sandbox) ResolveIP(ip string) string {
  344. var svc string
  345. logrus.Debugf("IP To resolve %v", ip)
  346. for _, ep := range sb.Endpoints() {
  347. n := ep.getNetwork()
  348. svc = n.ResolveIP(ip)
  349. if len(svc) != 0 {
  350. return svc
  351. }
  352. }
  353. return svc
  354. }
  355. func (sb *Sandbox) ExecFunc(f func()) error {
  356. sb.mu.Lock()
  357. osSbox := sb.osSbox
  358. sb.mu.Unlock()
  359. if osSbox != nil {
  360. return osSbox.InvokeFunc(f)
  361. }
  362. return fmt.Errorf("osl sandbox unavailable in ExecFunc for %v", sb.ContainerID())
  363. }
  364. // ResolveService returns all the backend details about the containers or hosts
  365. // backing a service. Its purpose is to satisfy an SRV query.
  366. func (sb *Sandbox) ResolveService(name string) ([]*net.SRV, []net.IP) {
  367. srv := []*net.SRV{}
  368. ip := []net.IP{}
  369. logrus.Debugf("Service name To resolve: %v", name)
  370. // There are DNS implementations that allow SRV queries for names not in
  371. // the format defined by RFC 2782. Hence specific validations checks are
  372. // not done
  373. parts := strings.Split(name, ".")
  374. if len(parts) < 3 {
  375. return nil, nil
  376. }
  377. for _, ep := range sb.Endpoints() {
  378. n := ep.getNetwork()
  379. srv, ip = n.ResolveService(name)
  380. if len(srv) > 0 {
  381. break
  382. }
  383. }
  384. return srv, ip
  385. }
  386. func getDynamicNwEndpoints(epList []*Endpoint) []*Endpoint {
  387. eps := []*Endpoint{}
  388. for _, ep := range epList {
  389. n := ep.getNetwork()
  390. if n.dynamic && !n.ingress {
  391. eps = append(eps, ep)
  392. }
  393. }
  394. return eps
  395. }
  396. func getIngressNwEndpoint(epList []*Endpoint) *Endpoint {
  397. for _, ep := range epList {
  398. n := ep.getNetwork()
  399. if n.ingress {
  400. return ep
  401. }
  402. }
  403. return nil
  404. }
  405. func getLocalNwEndpoints(epList []*Endpoint) []*Endpoint {
  406. eps := []*Endpoint{}
  407. for _, ep := range epList {
  408. n := ep.getNetwork()
  409. if !n.dynamic && !n.ingress {
  410. eps = append(eps, ep)
  411. }
  412. }
  413. return eps
  414. }
  415. func (sb *Sandbox) ResolveName(name string, ipType int) ([]net.IP, bool) {
  416. // Embedded server owns the docker network domain. Resolution should work
  417. // for both container_name and container_name.network_name
  418. // We allow '.' in service name and network name. For a name a.b.c.d the
  419. // following have to tried;
  420. // {a.b.c.d in the networks container is connected to}
  421. // {a.b.c in network d},
  422. // {a.b in network c.d},
  423. // {a in network b.c.d},
  424. logrus.Debugf("Name To resolve: %v", name)
  425. name = strings.TrimSuffix(name, ".")
  426. reqName := []string{name}
  427. networkName := []string{""}
  428. if strings.Contains(name, ".") {
  429. var i int
  430. dup := name
  431. for {
  432. if i = strings.LastIndex(dup, "."); i == -1 {
  433. break
  434. }
  435. networkName = append(networkName, name[i+1:])
  436. reqName = append(reqName, name[:i])
  437. dup = dup[:i]
  438. }
  439. }
  440. epList := sb.Endpoints()
  441. // In swarm mode services with exposed ports are connected to user overlay
  442. // network, ingress network and docker_gwbridge network. Name resolution
  443. // should prioritize returning the VIP/IPs on user overlay network.
  444. newList := []*Endpoint{}
  445. if !sb.controller.isDistributedControl() {
  446. newList = append(newList, getDynamicNwEndpoints(epList)...)
  447. ingressEP := getIngressNwEndpoint(epList)
  448. if ingressEP != nil {
  449. newList = append(newList, ingressEP)
  450. }
  451. newList = append(newList, getLocalNwEndpoints(epList)...)
  452. epList = newList
  453. }
  454. for i := 0; i < len(reqName); i++ {
  455. // First check for local container alias
  456. ip, ipv6Miss := sb.resolveName(reqName[i], networkName[i], epList, true, ipType)
  457. if ip != nil {
  458. return ip, false
  459. }
  460. if ipv6Miss {
  461. return ip, ipv6Miss
  462. }
  463. // Resolve the actual container name
  464. ip, ipv6Miss = sb.resolveName(reqName[i], networkName[i], epList, false, ipType)
  465. if ip != nil {
  466. return ip, false
  467. }
  468. if ipv6Miss {
  469. return ip, ipv6Miss
  470. }
  471. }
  472. return nil, false
  473. }
  474. func (sb *Sandbox) resolveName(req string, networkName string, epList []*Endpoint, alias bool, ipType int) ([]net.IP, bool) {
  475. var ipv6Miss bool
  476. for _, ep := range epList {
  477. name := req
  478. n := ep.getNetwork()
  479. if networkName != "" && networkName != n.Name() {
  480. continue
  481. }
  482. if alias {
  483. if ep.aliases == nil {
  484. continue
  485. }
  486. var ok bool
  487. ep.mu.Lock()
  488. name, ok = ep.aliases[req]
  489. ep.mu.Unlock()
  490. if !ok {
  491. continue
  492. }
  493. } else {
  494. // If it is a regular lookup and if the requested name is an alias
  495. // don't perform a svc lookup for this endpoint.
  496. ep.mu.Lock()
  497. if _, ok := ep.aliases[req]; ok {
  498. ep.mu.Unlock()
  499. continue
  500. }
  501. ep.mu.Unlock()
  502. }
  503. ip, miss := n.ResolveName(name, ipType)
  504. if ip != nil {
  505. return ip, false
  506. }
  507. if miss {
  508. ipv6Miss = miss
  509. }
  510. }
  511. return nil, ipv6Miss
  512. }
  513. // SetKey updates the Sandbox Key.
  514. func (sb *Sandbox) SetKey(basePath string) error {
  515. start := time.Now()
  516. defer func() {
  517. logrus.Debugf("sandbox set key processing took %s for container %s", time.Since(start), sb.ContainerID())
  518. }()
  519. if basePath == "" {
  520. return types.BadRequestErrorf("invalid sandbox key")
  521. }
  522. sb.mu.Lock()
  523. if sb.inDelete {
  524. sb.mu.Unlock()
  525. return types.ForbiddenErrorf("failed to SetKey: sandbox %q delete in progress", sb.id)
  526. }
  527. oldosSbox := sb.osSbox
  528. sb.mu.Unlock()
  529. if oldosSbox != nil {
  530. // If we already have an OS sandbox, release the network resources from that
  531. // and destroy the OS snab. We are moving into a new home further down. Note that none
  532. // of the network resources gets destroyed during the move.
  533. sb.releaseOSSbox()
  534. }
  535. osSbox, err := osl.GetSandboxForExternalKey(basePath, sb.Key())
  536. if err != nil {
  537. return err
  538. }
  539. sb.mu.Lock()
  540. sb.osSbox = osSbox
  541. sb.mu.Unlock()
  542. // If the resolver was setup before stop it and set it up in the
  543. // new osl sandbox.
  544. if oldosSbox != nil && sb.resolver != nil {
  545. sb.resolver.Stop()
  546. if err := sb.osSbox.InvokeFunc(sb.resolver.SetupFunc(0)); err == nil {
  547. if err := sb.resolver.Start(); err != nil {
  548. logrus.Errorf("Resolver Start failed for container %s, %q", sb.ContainerID(), err)
  549. }
  550. } else {
  551. logrus.Errorf("Resolver Setup Function failed for container %s, %q", sb.ContainerID(), err)
  552. }
  553. }
  554. for _, ep := range sb.Endpoints() {
  555. if err = sb.populateNetworkResources(ep); err != nil {
  556. return err
  557. }
  558. }
  559. return nil
  560. }
  561. // EnableService makes a managed container's service available by adding the
  562. // endpoint to the service load balancer and service discovery.
  563. func (sb *Sandbox) EnableService() (err error) {
  564. logrus.Debugf("EnableService %s START", sb.containerID)
  565. defer func() {
  566. if err != nil {
  567. if err2 := sb.DisableService(); err2 != nil {
  568. logrus.WithError(err2).WithField("origError", err).Error("Error while disabling service after original error")
  569. }
  570. }
  571. }()
  572. for _, ep := range sb.Endpoints() {
  573. if !ep.isServiceEnabled() {
  574. if err := ep.addServiceInfoToCluster(sb); err != nil {
  575. return fmt.Errorf("could not update state for endpoint %s into cluster: %v", ep.Name(), err)
  576. }
  577. ep.enableService()
  578. }
  579. }
  580. logrus.Debugf("EnableService %s DONE", sb.containerID)
  581. return nil
  582. }
  583. // DisableService removes a managed container's endpoints from the load balancer
  584. // and service discovery.
  585. func (sb *Sandbox) DisableService() (err error) {
  586. logrus.Debugf("DisableService %s START", sb.containerID)
  587. failedEps := []string{}
  588. defer func() {
  589. if len(failedEps) > 0 {
  590. err = fmt.Errorf("failed to disable service on sandbox:%s, for endpoints %s", sb.ID(), strings.Join(failedEps, ","))
  591. }
  592. }()
  593. for _, ep := range sb.Endpoints() {
  594. if ep.isServiceEnabled() {
  595. if err := ep.deleteServiceInfoFromCluster(sb, false, "DisableService"); err != nil {
  596. failedEps = append(failedEps, ep.Name())
  597. logrus.Warnf("failed update state for endpoint %s into cluster: %v", ep.Name(), err)
  598. }
  599. ep.disableService()
  600. }
  601. }
  602. logrus.Debugf("DisableService %s DONE", sb.containerID)
  603. return nil
  604. }
  605. func releaseOSSboxResources(osSbox osl.Sandbox, ep *Endpoint) {
  606. for _, i := range osSbox.Info().Interfaces() {
  607. // Only remove the interfaces owned by this endpoint from the sandbox.
  608. if ep.hasInterface(i.SrcName()) {
  609. if err := i.Remove(); err != nil {
  610. logrus.Debugf("Remove interface %s failed: %v", i.SrcName(), err)
  611. }
  612. }
  613. }
  614. ep.mu.Lock()
  615. joinInfo := ep.joinInfo
  616. vip := ep.virtualIP
  617. lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR
  618. ep.mu.Unlock()
  619. if len(vip) > 0 && lbModeIsDSR {
  620. ipNet := &net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)}
  621. if err := osSbox.RemoveAliasIP(osSbox.GetLoopbackIfaceName(), ipNet); err != nil {
  622. logrus.WithError(err).Debugf("failed to remove virtual ip %v to loopback", ipNet)
  623. }
  624. }
  625. if joinInfo == nil {
  626. return
  627. }
  628. // Remove non-interface routes.
  629. for _, r := range joinInfo.StaticRoutes {
  630. if err := osSbox.RemoveStaticRoute(r); err != nil {
  631. logrus.Debugf("Remove route failed: %v", err)
  632. }
  633. }
  634. }
  635. func (sb *Sandbox) releaseOSSbox() {
  636. sb.mu.Lock()
  637. osSbox := sb.osSbox
  638. sb.osSbox = nil
  639. sb.mu.Unlock()
  640. if osSbox == nil {
  641. return
  642. }
  643. for _, ep := range sb.Endpoints() {
  644. releaseOSSboxResources(osSbox, ep)
  645. }
  646. if err := osSbox.Destroy(); err != nil {
  647. logrus.WithError(err).Error("Error destroying os sandbox")
  648. }
  649. }
  650. func (sb *Sandbox) restoreOslSandbox() error {
  651. var routes []*types.StaticRoute
  652. // restore osl sandbox
  653. Ifaces := make(map[string][]osl.IfaceOption)
  654. for _, ep := range sb.endpoints {
  655. ep.mu.Lock()
  656. joinInfo := ep.joinInfo
  657. i := ep.iface
  658. ep.mu.Unlock()
  659. if i == nil {
  660. logrus.Errorf("error restoring endpoint %s for container %s", ep.Name(), sb.ContainerID())
  661. continue
  662. }
  663. ifaceOptions := []osl.IfaceOption{
  664. sb.osSbox.InterfaceOptions().Address(i.addr),
  665. sb.osSbox.InterfaceOptions().Routes(i.routes),
  666. }
  667. if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
  668. ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))
  669. }
  670. if i.mac != nil {
  671. ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac))
  672. }
  673. if len(i.llAddrs) != 0 {
  674. ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs))
  675. }
  676. Ifaces[i.srcName+i.dstPrefix] = ifaceOptions
  677. if joinInfo != nil {
  678. routes = append(routes, joinInfo.StaticRoutes...)
  679. }
  680. if ep.needResolver() {
  681. sb.startResolver(true)
  682. }
  683. }
  684. gwep := sb.getGatewayEndpoint()
  685. if gwep == nil {
  686. return nil
  687. }
  688. // restore osl sandbox
  689. return sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6)
  690. }
  691. func (sb *Sandbox) populateNetworkResources(ep *Endpoint) error {
  692. sb.mu.Lock()
  693. if sb.osSbox == nil {
  694. sb.mu.Unlock()
  695. return nil
  696. }
  697. inDelete := sb.inDelete
  698. sb.mu.Unlock()
  699. ep.mu.Lock()
  700. joinInfo := ep.joinInfo
  701. i := ep.iface
  702. lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR
  703. ep.mu.Unlock()
  704. if ep.needResolver() {
  705. sb.startResolver(false)
  706. }
  707. if i != nil && i.srcName != "" {
  708. var ifaceOptions []osl.IfaceOption
  709. ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes))
  710. if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
  711. ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))
  712. }
  713. if len(i.llAddrs) != 0 {
  714. ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs))
  715. }
  716. if i.mac != nil {
  717. ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac))
  718. }
  719. if err := sb.osSbox.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil {
  720. return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err)
  721. }
  722. if len(ep.virtualIP) > 0 && lbModeIsDSR {
  723. if sb.loadBalancerNID == "" {
  724. if err := sb.osSbox.DisableARPForVIP(i.srcName); err != nil {
  725. return fmt.Errorf("failed disable ARP for VIP: %v", err)
  726. }
  727. }
  728. ipNet := &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)}
  729. if err := sb.osSbox.AddAliasIP(sb.osSbox.GetLoopbackIfaceName(), ipNet); err != nil {
  730. return fmt.Errorf("failed to add virtual ip %v to loopback: %v", ipNet, err)
  731. }
  732. }
  733. }
  734. if joinInfo != nil {
  735. // Set up non-interface routes.
  736. for _, r := range joinInfo.StaticRoutes {
  737. if err := sb.osSbox.AddStaticRoute(r); err != nil {
  738. return fmt.Errorf("failed to add static route %s: %v", r.Destination.String(), err)
  739. }
  740. }
  741. }
  742. if ep == sb.getGatewayEndpoint() {
  743. if err := sb.updateGateway(ep); err != nil {
  744. return err
  745. }
  746. }
  747. // Make sure to add the endpoint to the populated endpoint set
  748. // before populating loadbalancers.
  749. sb.mu.Lock()
  750. sb.populatedEndpoints[ep.ID()] = struct{}{}
  751. sb.mu.Unlock()
  752. // Populate load balancer only after updating all the other
  753. // information including gateway and other routes so that
  754. // loadbalancers are populated all the network state is in
  755. // place in the sandbox.
  756. sb.populateLoadBalancers(ep)
  757. // Only update the store if we did not come here as part of
  758. // sandbox delete. If we came here as part of delete then do
  759. // not bother updating the store. The sandbox object will be
  760. // deleted anyway
  761. if !inDelete {
  762. return sb.storeUpdate()
  763. }
  764. return nil
  765. }
  766. func (sb *Sandbox) clearNetworkResources(origEp *Endpoint) error {
  767. ep := sb.getEndpoint(origEp.id)
  768. if ep == nil {
  769. return fmt.Errorf("could not find the sandbox endpoint data for endpoint %s",
  770. origEp.id)
  771. }
  772. sb.mu.Lock()
  773. osSbox := sb.osSbox
  774. inDelete := sb.inDelete
  775. sb.mu.Unlock()
  776. if osSbox != nil {
  777. releaseOSSboxResources(osSbox, ep)
  778. }
  779. sb.mu.Lock()
  780. delete(sb.populatedEndpoints, ep.ID())
  781. if len(sb.endpoints) == 0 {
  782. // sb.endpoints should never be empty and this is unexpected error condition
  783. // We log an error message to note this down for debugging purposes.
  784. logrus.Errorf("No endpoints in sandbox while trying to remove endpoint %s", ep.Name())
  785. sb.mu.Unlock()
  786. return nil
  787. }
  788. var (
  789. gwepBefore, gwepAfter *Endpoint
  790. index = -1
  791. )
  792. for i, e := range sb.endpoints {
  793. if e == ep {
  794. index = i
  795. }
  796. if len(e.Gateway()) > 0 && gwepBefore == nil {
  797. gwepBefore = e
  798. }
  799. if index != -1 && gwepBefore != nil {
  800. break
  801. }
  802. }
  803. if index == -1 {
  804. logrus.Warnf("Endpoint %s has already been deleted", ep.Name())
  805. sb.mu.Unlock()
  806. return nil
  807. }
  808. sb.removeEndpointRaw(ep)
  809. for _, e := range sb.endpoints {
  810. if len(e.Gateway()) > 0 {
  811. gwepAfter = e
  812. break
  813. }
  814. }
  815. delete(sb.epPriority, ep.ID())
  816. sb.mu.Unlock()
  817. if gwepAfter != nil && gwepBefore != gwepAfter {
  818. if err := sb.updateGateway(gwepAfter); err != nil {
  819. return err
  820. }
  821. }
  822. // Only update the store if we did not come here as part of
  823. // sandbox delete. If we came here as part of delete then do
  824. // not bother updating the store. The sandbox object will be
  825. // deleted anyway
  826. if !inDelete {
  827. return sb.storeUpdate()
  828. }
  829. return nil
  830. }
  831. // joinLeaveStart waits to ensure there are no joins or leaves in progress and
  832. // marks this join/leave in progress without race
  833. func (sb *Sandbox) joinLeaveStart() {
  834. sb.mu.Lock()
  835. defer sb.mu.Unlock()
  836. for sb.joinLeaveDone != nil {
  837. joinLeaveDone := sb.joinLeaveDone
  838. sb.mu.Unlock()
  839. <-joinLeaveDone
  840. sb.mu.Lock()
  841. }
  842. sb.joinLeaveDone = make(chan struct{})
  843. }
  844. // joinLeaveEnd marks the end of this join/leave operation and
  845. // signals the same without race to other join and leave waiters
  846. func (sb *Sandbox) joinLeaveEnd() {
  847. sb.mu.Lock()
  848. defer sb.mu.Unlock()
  849. if sb.joinLeaveDone != nil {
  850. close(sb.joinLeaveDone)
  851. sb.joinLeaveDone = nil
  852. }
  853. }
  854. // OptionHostname function returns an option setter for hostname option to
  855. // be passed to NewSandbox method.
  856. func OptionHostname(name string) SandboxOption {
  857. return func(sb *Sandbox) {
  858. sb.config.hostName = name
  859. }
  860. }
  861. // OptionDomainname function returns an option setter for domainname option to
  862. // be passed to NewSandbox method.
  863. func OptionDomainname(name string) SandboxOption {
  864. return func(sb *Sandbox) {
  865. sb.config.domainName = name
  866. }
  867. }
  868. // OptionHostsPath function returns an option setter for hostspath option to
  869. // be passed to NewSandbox method.
  870. func OptionHostsPath(path string) SandboxOption {
  871. return func(sb *Sandbox) {
  872. sb.config.hostsPath = path
  873. }
  874. }
  875. // OptionOriginHostsPath function returns an option setter for origin hosts file path
  876. // to be passed to NewSandbox method.
  877. func OptionOriginHostsPath(path string) SandboxOption {
  878. return func(sb *Sandbox) {
  879. sb.config.originHostsPath = path
  880. }
  881. }
  882. // OptionExtraHost function returns an option setter for extra /etc/hosts options
  883. // which is a name and IP as strings.
  884. func OptionExtraHost(name string, IP string) SandboxOption {
  885. return func(sb *Sandbox) {
  886. sb.config.extraHosts = append(sb.config.extraHosts, extraHost{name: name, IP: IP})
  887. }
  888. }
  889. // OptionParentUpdate function returns an option setter for parent container
  890. // which needs to update the IP address for the linked container.
  891. func OptionParentUpdate(cid string, name, ip string) SandboxOption {
  892. return func(sb *Sandbox) {
  893. sb.config.parentUpdates = append(sb.config.parentUpdates, parentUpdate{cid: cid, name: name, ip: ip})
  894. }
  895. }
  896. // OptionResolvConfPath function returns an option setter for resolvconfpath option to
  897. // be passed to net container methods.
  898. func OptionResolvConfPath(path string) SandboxOption {
  899. return func(sb *Sandbox) {
  900. sb.config.resolvConfPath = path
  901. }
  902. }
  903. // OptionOriginResolvConfPath function returns an option setter to set the path to the
  904. // origin resolv.conf file to be passed to net container methods.
  905. func OptionOriginResolvConfPath(path string) SandboxOption {
  906. return func(sb *Sandbox) {
  907. sb.config.originResolvConfPath = path
  908. }
  909. }
  910. // OptionDNS function returns an option setter for dns entry option to
  911. // be passed to container Create method.
  912. func OptionDNS(dns string) SandboxOption {
  913. return func(sb *Sandbox) {
  914. sb.config.dnsList = append(sb.config.dnsList, dns)
  915. }
  916. }
  917. // OptionDNSSearch function returns an option setter for dns search entry option to
  918. // be passed to container Create method.
  919. func OptionDNSSearch(search string) SandboxOption {
  920. return func(sb *Sandbox) {
  921. sb.config.dnsSearchList = append(sb.config.dnsSearchList, search)
  922. }
  923. }
  924. // OptionDNSOptions function returns an option setter for dns options entry option to
  925. // be passed to container Create method.
  926. func OptionDNSOptions(options string) SandboxOption {
  927. return func(sb *Sandbox) {
  928. sb.config.dnsOptionsList = append(sb.config.dnsOptionsList, options)
  929. }
  930. }
  931. // OptionUseDefaultSandbox function returns an option setter for using default sandbox
  932. // (host namespace) to be passed to container Create method.
  933. func OptionUseDefaultSandbox() SandboxOption {
  934. return func(sb *Sandbox) {
  935. sb.config.useDefaultSandBox = true
  936. }
  937. }
  938. // OptionUseExternalKey function returns an option setter for using provided namespace
  939. // instead of creating one.
  940. func OptionUseExternalKey() SandboxOption {
  941. return func(sb *Sandbox) {
  942. sb.config.useExternalKey = true
  943. }
  944. }
  945. // OptionGeneric function returns an option setter for Generic configuration
  946. // that is not managed by libNetwork but can be used by the Drivers during the call to
  947. // net container creation method. Container Labels are a good example.
  948. func OptionGeneric(generic map[string]interface{}) SandboxOption {
  949. return func(sb *Sandbox) {
  950. if sb.config.generic == nil {
  951. sb.config.generic = make(map[string]interface{}, len(generic))
  952. }
  953. for k, v := range generic {
  954. sb.config.generic[k] = v
  955. }
  956. }
  957. }
  958. // OptionExposedPorts function returns an option setter for the container exposed
  959. // ports option to be passed to container Create method.
  960. func OptionExposedPorts(exposedPorts []types.TransportPort) SandboxOption {
  961. return func(sb *Sandbox) {
  962. if sb.config.generic == nil {
  963. sb.config.generic = make(map[string]interface{})
  964. }
  965. // Defensive copy
  966. eps := make([]types.TransportPort, len(exposedPorts))
  967. copy(eps, exposedPorts)
  968. // Store endpoint label and in generic because driver needs it
  969. sb.config.exposedPorts = eps
  970. sb.config.generic[netlabel.ExposedPorts] = eps
  971. }
  972. }
  973. // OptionPortMapping function returns an option setter for the mapping
  974. // ports option to be passed to container Create method.
  975. func OptionPortMapping(portBindings []types.PortBinding) SandboxOption {
  976. return func(sb *Sandbox) {
  977. if sb.config.generic == nil {
  978. sb.config.generic = make(map[string]interface{})
  979. }
  980. // Store a copy of the bindings as generic data to pass to the driver
  981. pbs := make([]types.PortBinding, len(portBindings))
  982. copy(pbs, portBindings)
  983. sb.config.generic[netlabel.PortMap] = pbs
  984. }
  985. }
  986. // OptionIngress function returns an option setter for marking a
  987. // sandbox as the controller's ingress sandbox.
  988. func OptionIngress() SandboxOption {
  989. return func(sb *Sandbox) {
  990. sb.ingress = true
  991. sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeIngress)
  992. }
  993. }
  994. // OptionLoadBalancer function returns an option setter for marking a
  995. // sandbox as a load balancer sandbox.
  996. func OptionLoadBalancer(nid string) SandboxOption {
  997. return func(sb *Sandbox) {
  998. sb.loadBalancerNID = nid
  999. sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeLoadBalancer)
  1000. }
  1001. }
  1002. // <=> Returns true if a < b, false if a > b and advances to next level if a == b
  1003. // epi.prio <=> epj.prio # 2 < 1
  1004. // epi.gw <=> epj.gw # non-gw < gw
  1005. // epi.internal <=> epj.internal # non-internal < internal
  1006. // epi.joininfo <=> epj.joininfo # ipv6 < ipv4
  1007. // epi.name <=> epj.name # bar < foo
  1008. func (epi *Endpoint) Less(epj *Endpoint) bool {
  1009. var (
  1010. prioi, prioj int
  1011. )
  1012. sbi, _ := epi.getSandbox()
  1013. sbj, _ := epj.getSandbox()
  1014. // Prio defaults to 0
  1015. if sbi != nil {
  1016. prioi = sbi.epPriority[epi.ID()]
  1017. }
  1018. if sbj != nil {
  1019. prioj = sbj.epPriority[epj.ID()]
  1020. }
  1021. if prioi != prioj {
  1022. return prioi > prioj
  1023. }
  1024. gwi := epi.endpointInGWNetwork()
  1025. gwj := epj.endpointInGWNetwork()
  1026. if gwi != gwj {
  1027. return gwj
  1028. }
  1029. inti := epi.getNetwork().Internal()
  1030. intj := epj.getNetwork().Internal()
  1031. if inti != intj {
  1032. return intj
  1033. }
  1034. jii := 0
  1035. if epi.joinInfo != nil {
  1036. if epi.joinInfo.gw != nil {
  1037. jii = jii + 1
  1038. }
  1039. if epi.joinInfo.gw6 != nil {
  1040. jii = jii + 2
  1041. }
  1042. }
  1043. jij := 0
  1044. if epj.joinInfo != nil {
  1045. if epj.joinInfo.gw != nil {
  1046. jij = jij + 1
  1047. }
  1048. if epj.joinInfo.gw6 != nil {
  1049. jij = jij + 2
  1050. }
  1051. }
  1052. if jii != jij {
  1053. return jii > jij
  1054. }
  1055. return epi.network.Name() < epj.network.Name()
  1056. }
  1057. func (sb *Sandbox) NdotsSet() bool {
  1058. return sb.ndotsSet
  1059. }