service_common.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381
  1. // +build linux windows
  2. package libnetwork
  3. import (
  4. "net"
  5. "github.com/docker/libnetwork/common"
  6. "github.com/sirupsen/logrus"
  7. )
  8. func (c *controller) addEndpointNameResolution(svcName, svcID, nID, eID, containerName string, vip net.IP, serviceAliases, taskAliases []string, ip net.IP, addService bool, method string) error {
  9. n, err := c.NetworkByID(nID)
  10. if err != nil {
  11. return err
  12. }
  13. logrus.Debugf("addEndpointNameResolution %s %s add_service:%t sAliases:%v tAliases:%v", eID, svcName, addService, serviceAliases, taskAliases)
  14. // Add container resolution mappings
  15. c.addContainerNameResolution(nID, eID, containerName, taskAliases, ip, method)
  16. serviceID := svcID
  17. if serviceID == "" {
  18. // This is the case of a normal container not part of a service
  19. serviceID = eID
  20. }
  21. // Add endpoint IP to special "tasks.svc_name" so that the applications have access to DNS RR.
  22. n.(*network).addSvcRecords(eID, "tasks."+svcName, serviceID, ip, nil, false, method)
  23. for _, alias := range serviceAliases {
  24. n.(*network).addSvcRecords(eID, "tasks."+alias, serviceID, ip, nil, false, method)
  25. }
  26. // Add service name to vip in DNS, if vip is valid. Otherwise resort to DNS RR
  27. if len(vip) == 0 {
  28. n.(*network).addSvcRecords(eID, svcName, serviceID, ip, nil, false, method)
  29. for _, alias := range serviceAliases {
  30. n.(*network).addSvcRecords(eID, alias, serviceID, ip, nil, false, method)
  31. }
  32. }
  33. if addService && len(vip) != 0 {
  34. n.(*network).addSvcRecords(eID, svcName, serviceID, vip, nil, false, method)
  35. for _, alias := range serviceAliases {
  36. n.(*network).addSvcRecords(eID, alias, serviceID, vip, nil, false, method)
  37. }
  38. }
  39. return nil
  40. }
  41. func (c *controller) addContainerNameResolution(nID, eID, containerName string, taskAliases []string, ip net.IP, method string) error {
  42. n, err := c.NetworkByID(nID)
  43. if err != nil {
  44. return err
  45. }
  46. logrus.Debugf("addContainerNameResolution %s %s", eID, containerName)
  47. // Add resolution for container name
  48. n.(*network).addSvcRecords(eID, containerName, eID, ip, nil, true, method)
  49. // Add resolution for taskaliases
  50. for _, alias := range taskAliases {
  51. n.(*network).addSvcRecords(eID, alias, eID, ip, nil, true, method)
  52. }
  53. return nil
  54. }
  55. func (c *controller) deleteEndpointNameResolution(svcName, svcID, nID, eID, containerName string, vip net.IP, serviceAliases, taskAliases []string, ip net.IP, rmService, multipleEntries bool, method string) error {
  56. n, err := c.NetworkByID(nID)
  57. if err != nil {
  58. return err
  59. }
  60. logrus.Debugf("deleteEndpointNameResolution %s %s rm_service:%t suppress:%t sAliases:%v tAliases:%v", eID, svcName, rmService, multipleEntries, serviceAliases, taskAliases)
  61. // Delete container resolution mappings
  62. c.delContainerNameResolution(nID, eID, containerName, taskAliases, ip, method)
  63. serviceID := svcID
  64. if serviceID == "" {
  65. // This is the case of a normal container not part of a service
  66. serviceID = eID
  67. }
  68. // Delete the special "tasks.svc_name" backend record.
  69. if !multipleEntries {
  70. n.(*network).deleteSvcRecords(eID, "tasks."+svcName, serviceID, ip, nil, false, method)
  71. for _, alias := range serviceAliases {
  72. n.(*network).deleteSvcRecords(eID, "tasks."+alias, serviceID, ip, nil, false, method)
  73. }
  74. }
  75. // If we are doing DNS RR delete the endpoint IP from DNS record right away.
  76. if !multipleEntries && len(vip) == 0 {
  77. n.(*network).deleteSvcRecords(eID, svcName, serviceID, ip, nil, false, method)
  78. for _, alias := range serviceAliases {
  79. n.(*network).deleteSvcRecords(eID, alias, serviceID, ip, nil, false, method)
  80. }
  81. }
  82. // Remove the DNS record for VIP only if we are removing the service
  83. if rmService && len(vip) != 0 && !multipleEntries {
  84. n.(*network).deleteSvcRecords(eID, svcName, serviceID, vip, nil, false, method)
  85. for _, alias := range serviceAliases {
  86. n.(*network).deleteSvcRecords(eID, alias, serviceID, vip, nil, false, method)
  87. }
  88. }
  89. return nil
  90. }
  91. func (c *controller) delContainerNameResolution(nID, eID, containerName string, taskAliases []string, ip net.IP, method string) error {
  92. n, err := c.NetworkByID(nID)
  93. if err != nil {
  94. return err
  95. }
  96. logrus.Debugf("delContainerNameResolution %s %s", eID, containerName)
  97. // Delete resolution for container name
  98. n.(*network).deleteSvcRecords(eID, containerName, eID, ip, nil, true, method)
  99. // Delete resolution for taskaliases
  100. for _, alias := range taskAliases {
  101. n.(*network).deleteSvcRecords(eID, alias, eID, ip, nil, true, method)
  102. }
  103. return nil
  104. }
  105. func newService(name string, id string, ingressPorts []*PortConfig, serviceAliases []string) *service {
  106. return &service{
  107. name: name,
  108. id: id,
  109. ingressPorts: ingressPorts,
  110. loadBalancers: make(map[string]*loadBalancer),
  111. aliases: serviceAliases,
  112. ipToEndpoint: common.NewSetMatrix(),
  113. }
  114. }
  115. func (c *controller) getLBIndex(sid, nid string, ingressPorts []*PortConfig) int {
  116. skey := serviceKey{
  117. id: sid,
  118. ports: portConfigs(ingressPorts).String(),
  119. }
  120. c.Lock()
  121. s, ok := c.serviceBindings[skey]
  122. c.Unlock()
  123. if !ok {
  124. return 0
  125. }
  126. s.Lock()
  127. lb := s.loadBalancers[nid]
  128. s.Unlock()
  129. return int(lb.fwMark)
  130. }
  131. func (c *controller) cleanupServiceBindings(cleanupNID string) {
  132. var cleanupFuncs []func()
  133. logrus.Debugf("cleanupServiceBindings for %s", cleanupNID)
  134. c.Lock()
  135. services := make([]*service, 0, len(c.serviceBindings))
  136. for _, s := range c.serviceBindings {
  137. services = append(services, s)
  138. }
  139. c.Unlock()
  140. for _, s := range services {
  141. s.Lock()
  142. // Skip the serviceBindings that got deleted
  143. if s.deleted {
  144. s.Unlock()
  145. continue
  146. }
  147. for nid, lb := range s.loadBalancers {
  148. if cleanupNID != "" && nid != cleanupNID {
  149. continue
  150. }
  151. // The network is being deleted, erase all the associated service discovery records
  152. // TODO(fcrisciani) separate the Load Balancer from the Service discovery, this operation
  153. // can be done safely here, but the rmServiceBinding is still keeping consistency in the
  154. // data structures that are tracking the endpoint to IP mapping.
  155. c.Lock()
  156. logrus.Debugf("cleanupServiceBindings erasing the svcRecords for %s", nid)
  157. delete(c.svcRecords, nid)
  158. c.Unlock()
  159. for eid, ip := range lb.backEnds {
  160. epID := eid
  161. epIP := ip
  162. service := s
  163. loadBalancer := lb
  164. networkID := nid
  165. cleanupFuncs = append(cleanupFuncs, func() {
  166. // ContainerName and taskAliases are not available here, this is still fine because the Service discovery
  167. // cleanup already happened before. The only thing that rmServiceBinding is still doing here a part from the Load
  168. // Balancer bookeeping, is to keep consistent the mapping of endpoint to IP.
  169. if err := c.rmServiceBinding(service.name, service.id, networkID, epID, "", loadBalancer.vip,
  170. service.ingressPorts, service.aliases, []string{}, epIP, "cleanupServiceBindings", false); err != nil {
  171. logrus.Errorf("Failed to remove service bindings for service %s network %s endpoint %s while cleanup: %v",
  172. service.id, networkID, epID, err)
  173. }
  174. })
  175. }
  176. }
  177. s.Unlock()
  178. }
  179. for _, f := range cleanupFuncs {
  180. f()
  181. }
  182. }
  183. func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName string, vip net.IP, ingressPorts []*PortConfig, serviceAliases, taskAliases []string, ip net.IP, method string) error {
  184. var addService bool
  185. n, err := c.NetworkByID(nID)
  186. if err != nil {
  187. return err
  188. }
  189. skey := serviceKey{
  190. id: svcID,
  191. ports: portConfigs(ingressPorts).String(),
  192. }
  193. var s *service
  194. for {
  195. c.Lock()
  196. var ok bool
  197. s, ok = c.serviceBindings[skey]
  198. if !ok {
  199. // Create a new service if we are seeing this service
  200. // for the first time.
  201. s = newService(svcName, svcID, ingressPorts, serviceAliases)
  202. c.serviceBindings[skey] = s
  203. }
  204. c.Unlock()
  205. s.Lock()
  206. if !s.deleted {
  207. // ok the object is good to be used
  208. break
  209. }
  210. s.Unlock()
  211. }
  212. logrus.Debugf("addServiceBinding from %s START for %s %s p:%p nid:%s skey:%v", method, svcName, eID, s, nID, skey)
  213. defer s.Unlock()
  214. lb, ok := s.loadBalancers[nID]
  215. if !ok {
  216. // Create a new load balancer if we are seeing this
  217. // network attachment on the service for the first
  218. // time.
  219. fwMarkCtrMu.Lock()
  220. lb = &loadBalancer{
  221. vip: vip,
  222. fwMark: fwMarkCtr,
  223. backEnds: make(map[string]net.IP),
  224. service: s,
  225. }
  226. fwMarkCtr++
  227. fwMarkCtrMu.Unlock()
  228. s.loadBalancers[nID] = lb
  229. addService = true
  230. }
  231. lb.backEnds[eID] = ip
  232. ok, entries := s.assignIPToEndpoint(ip.String(), eID)
  233. if !ok || entries > 1 {
  234. setStr, b := s.printIPToEndpoint(ip.String())
  235. logrus.Warnf("addServiceBinding %s possible trainsient state ok:%t entries:%d set:%t %s", eID, ok, entries, b, setStr)
  236. }
  237. // Add loadbalancer service and backend in all sandboxes in
  238. // the network only if vip is valid.
  239. if len(vip) != 0 {
  240. n.(*network).addLBBackend(ip, vip, lb, ingressPorts)
  241. }
  242. // Add the appropriate name resolutions
  243. c.addEndpointNameResolution(svcName, svcID, nID, eID, containerName, vip, serviceAliases, taskAliases, ip, addService, "addServiceBinding")
  244. logrus.Debugf("addServiceBinding from %s END for %s %s", method, svcName, eID)
  245. return nil
  246. }
  247. func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName string, vip net.IP, ingressPorts []*PortConfig, serviceAliases []string, taskAliases []string, ip net.IP, method string, deleteSvcRecords bool) error {
  248. var rmService bool
  249. n, err := c.NetworkByID(nID)
  250. if err != nil {
  251. return err
  252. }
  253. skey := serviceKey{
  254. id: svcID,
  255. ports: portConfigs(ingressPorts).String(),
  256. }
  257. c.Lock()
  258. s, ok := c.serviceBindings[skey]
  259. c.Unlock()
  260. if !ok {
  261. logrus.Warnf("rmServiceBinding %s %s %s aborted c.serviceBindings[skey] !ok", method, svcName, eID)
  262. return nil
  263. }
  264. s.Lock()
  265. defer s.Unlock()
  266. logrus.Debugf("rmServiceBinding from %s START for %s %s p:%p nid:%s sKey:%v deleteSvc:%t", method, svcName, eID, s, nID, skey, deleteSvcRecords)
  267. lb, ok := s.loadBalancers[nID]
  268. if !ok {
  269. logrus.Warnf("rmServiceBinding %s %s %s aborted s.loadBalancers[nid] !ok", method, svcName, eID)
  270. return nil
  271. }
  272. _, ok = lb.backEnds[eID]
  273. if !ok {
  274. logrus.Warnf("rmServiceBinding %s %s %s aborted lb.backEnds[eid] !ok", method, svcName, eID)
  275. return nil
  276. }
  277. delete(lb.backEnds, eID)
  278. if len(lb.backEnds) == 0 {
  279. // All the backends for this service have been
  280. // removed. Time to remove the load balancer and also
  281. // remove the service entry in IPVS.
  282. rmService = true
  283. delete(s.loadBalancers, nID)
  284. logrus.Debugf("rmServiceBinding %s delete %s, p:%p in loadbalancers len:%d", eID, nID, lb, len(s.loadBalancers))
  285. }
  286. ok, entries := s.removeIPToEndpoint(ip.String(), eID)
  287. if !ok || entries > 0 {
  288. setStr, b := s.printIPToEndpoint(ip.String())
  289. logrus.Warnf("rmServiceBinding %s possible trainsient state ok:%t entries:%d set:%t %s", eID, ok, entries, b, setStr)
  290. }
  291. // Remove loadbalancer service(if needed) and backend in all
  292. // sandboxes in the network only if the vip is valid.
  293. if len(vip) != 0 && entries == 0 {
  294. n.(*network).rmLBBackend(ip, vip, lb, ingressPorts, rmService)
  295. }
  296. // Delete the name resolutions
  297. if deleteSvcRecords {
  298. c.deleteEndpointNameResolution(svcName, svcID, nID, eID, containerName, vip, serviceAliases, taskAliases, ip, rmService, entries > 0, "rmServiceBinding")
  299. }
  300. if len(s.loadBalancers) == 0 {
  301. // All loadbalancers for the service removed. Time to
  302. // remove the service itself.
  303. c.Lock()
  304. // Mark the object as deleted so that the add won't use it wrongly
  305. s.deleted = true
  306. // NOTE The delete from the serviceBindings map has to be the last operation else we are allowing a race between this service
  307. // that is getting deleted and a new service that will be created if the entry is not anymore there
  308. delete(c.serviceBindings, skey)
  309. c.Unlock()
  310. }
  311. logrus.Debugf("rmServiceBinding from %s END for %s %s", method, svcName, eID)
  312. return nil
  313. }