setup_ip_tables.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429
  1. package bridge
  2. import (
  3. "errors"
  4. "fmt"
  5. "net"
  6. "github.com/docker/libnetwork/iptables"
  7. "github.com/sirupsen/logrus"
  8. "github.com/vishvananda/netlink"
  9. )
  10. // DockerChain: DOCKER iptable chain name
  11. const (
  12. DockerChain = "DOCKER"
  13. // Isolation between bridge networks is achieved in two stages by means
  14. // of the following two chains in the filter table. The first chain matches
  15. // on the source interface being a bridge network's bridge and the
  16. // destination being a different interface. A positive match leads to the
  17. // second isolation chain. No match returns to the parent chain. The second
  18. // isolation chain matches on destination interface being a bridge network's
  19. // bridge. A positive match identifies a packet originated from one bridge
  20. // network's bridge destined to another bridge network's bridge and will
  21. // result in the packet being dropped. No match returns to the parent chain.
  22. IsolationChain1 = "DOCKER-ISOLATION-STAGE-1"
  23. IsolationChain2 = "DOCKER-ISOLATION-STAGE-2"
  24. )
  25. func setupIPChains(config *configuration, version iptables.IPVersion) (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
  26. // Sanity check.
  27. if config.EnableIPTables == false {
  28. return nil, nil, nil, nil, errors.New("cannot create new chains, EnableIPTable is disabled")
  29. }
  30. hairpinMode := !config.EnableUserlandProxy
  31. iptable := iptables.GetIptable(version)
  32. natChain, err := iptable.NewChain(DockerChain, iptables.Nat, hairpinMode)
  33. if err != nil {
  34. return nil, nil, nil, nil, fmt.Errorf("failed to create NAT chain %s: %v", DockerChain, err)
  35. }
  36. defer func() {
  37. if err != nil {
  38. if err := iptable.RemoveExistingChain(DockerChain, iptables.Nat); err != nil {
  39. logrus.Warnf("failed on removing iptables NAT chain %s on cleanup: %v", DockerChain, err)
  40. }
  41. }
  42. }()
  43. filterChain, err := iptable.NewChain(DockerChain, iptables.Filter, false)
  44. if err != nil {
  45. return nil, nil, nil, nil, fmt.Errorf("failed to create FILTER chain %s: %v", DockerChain, err)
  46. }
  47. defer func() {
  48. if err != nil {
  49. if err := iptable.RemoveExistingChain(DockerChain, iptables.Filter); err != nil {
  50. logrus.Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", DockerChain, err)
  51. }
  52. }
  53. }()
  54. isolationChain1, err := iptable.NewChain(IsolationChain1, iptables.Filter, false)
  55. if err != nil {
  56. return nil, nil, nil, nil, fmt.Errorf("failed to create FILTER isolation chain: %v", err)
  57. }
  58. defer func() {
  59. if err != nil {
  60. if err := iptable.RemoveExistingChain(IsolationChain1, iptables.Filter); err != nil {
  61. logrus.Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", IsolationChain1, err)
  62. }
  63. }
  64. }()
  65. isolationChain2, err := iptable.NewChain(IsolationChain2, iptables.Filter, false)
  66. if err != nil {
  67. return nil, nil, nil, nil, fmt.Errorf("failed to create FILTER isolation chain: %v", err)
  68. }
  69. defer func() {
  70. if err != nil {
  71. if err := iptable.RemoveExistingChain(IsolationChain2, iptables.Filter); err != nil {
  72. logrus.Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", IsolationChain2, err)
  73. }
  74. }
  75. }()
  76. if err := iptable.AddReturnRule(IsolationChain1); err != nil {
  77. return nil, nil, nil, nil, err
  78. }
  79. if err := iptable.AddReturnRule(IsolationChain2); err != nil {
  80. return nil, nil, nil, nil, err
  81. }
  82. return natChain, filterChain, isolationChain1, isolationChain2, nil
  83. }
  84. func (n *bridgeNetwork) setupIP4Tables(config *networkConfiguration, i *bridgeInterface) error {
  85. d := n.driver
  86. d.Lock()
  87. driverConfig := d.config
  88. d.Unlock()
  89. // Sanity check.
  90. if !driverConfig.EnableIPTables {
  91. return errors.New("Cannot program chains, EnableIPTable is disabled")
  92. }
  93. maskedAddrv4 := &net.IPNet{
  94. IP: i.bridgeIPv4.IP.Mask(i.bridgeIPv4.Mask),
  95. Mask: i.bridgeIPv4.Mask,
  96. }
  97. return n.setupIPTables(iptables.IPv4, maskedAddrv4, config, i)
  98. }
  99. func (n *bridgeNetwork) setupIP6Tables(config *networkConfiguration, i *bridgeInterface) error {
  100. d := n.driver
  101. d.Lock()
  102. driverConfig := d.config
  103. d.Unlock()
  104. // Sanity check.
  105. if !driverConfig.EnableIP6Tables {
  106. return errors.New("Cannot program chains, EnableIP6Tables is disabled")
  107. }
  108. maskedAddrv6 := &net.IPNet{
  109. IP: i.bridgeIPv6.IP.Mask(i.bridgeIPv6.Mask),
  110. Mask: i.bridgeIPv6.Mask,
  111. }
  112. return n.setupIPTables(iptables.IPv6, maskedAddrv6, config, i)
  113. }
  114. func (n *bridgeNetwork) setupIPTables(ipVersion iptables.IPVersion, maskedAddr *net.IPNet, config *networkConfiguration, i *bridgeInterface) error {
  115. var err error
  116. d := n.driver
  117. d.Lock()
  118. driverConfig := d.config
  119. d.Unlock()
  120. // Pickup this configuration option from driver
  121. hairpinMode := !driverConfig.EnableUserlandProxy
  122. iptable := iptables.GetIptable(ipVersion)
  123. if config.Internal {
  124. if err = setupInternalNetworkRules(config.BridgeName, maskedAddr, config.EnableICC, true); err != nil {
  125. return fmt.Errorf("Failed to Setup IP tables: %s", err.Error())
  126. }
  127. n.registerIptCleanFunc(func() error {
  128. return setupInternalNetworkRules(config.BridgeName, maskedAddr, config.EnableICC, false)
  129. })
  130. } else {
  131. if err = setupIPTablesInternal(config.HostIP, config.BridgeName, maskedAddr, config.EnableICC, config.EnableIPMasquerade, hairpinMode, true); err != nil {
  132. return fmt.Errorf("Failed to Setup IP tables: %s", err.Error())
  133. }
  134. n.registerIptCleanFunc(func() error {
  135. return setupIPTablesInternal(config.HostIP, config.BridgeName, maskedAddr, config.EnableICC, config.EnableIPMasquerade, hairpinMode, false)
  136. })
  137. natChain, filterChain, _, _, err := n.getDriverChains(ipVersion)
  138. if err != nil {
  139. return fmt.Errorf("Failed to setup IP tables, cannot acquire chain info %s", err.Error())
  140. }
  141. err = iptable.ProgramChain(natChain, config.BridgeName, hairpinMode, true)
  142. if err != nil {
  143. return fmt.Errorf("Failed to program NAT chain: %s", err.Error())
  144. }
  145. err = iptable.ProgramChain(filterChain, config.BridgeName, hairpinMode, true)
  146. if err != nil {
  147. return fmt.Errorf("Failed to program FILTER chain: %s", err.Error())
  148. }
  149. n.registerIptCleanFunc(func() error {
  150. return iptable.ProgramChain(filterChain, config.BridgeName, hairpinMode, false)
  151. })
  152. if ipVersion == iptables.IPv4 {
  153. n.portMapper.SetIptablesChain(natChain, n.getNetworkBridgeName())
  154. } else {
  155. n.portMapperV6.SetIptablesChain(natChain, n.getNetworkBridgeName())
  156. }
  157. }
  158. d.Lock()
  159. err = iptable.EnsureJumpRule("FORWARD", IsolationChain1)
  160. d.Unlock()
  161. return err
  162. }
  163. type iptRule struct {
  164. table iptables.Table
  165. chain string
  166. preArgs []string
  167. args []string
  168. }
  169. func setupIPTablesInternal(hostIP net.IP, bridgeIface string, addr *net.IPNet, icc, ipmasq, hairpin, enable bool) error {
  170. var (
  171. address = addr.String()
  172. skipDNAT = iptRule{table: iptables.Nat, chain: DockerChain, preArgs: []string{"-t", "nat"}, args: []string{"-i", bridgeIface, "-j", "RETURN"}}
  173. outRule = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "-j", "ACCEPT"}}
  174. natArgs []string
  175. hpNatArgs []string
  176. )
  177. // if hostIP is set use this address as the src-ip during SNAT
  178. if hostIP != nil {
  179. hostAddr := hostIP.String()
  180. natArgs = []string{"-s", address, "!", "-o", bridgeIface, "-j", "SNAT", "--to-source", hostAddr}
  181. hpNatArgs = []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", bridgeIface, "-j", "SNAT", "--to-source", hostAddr}
  182. // Else use MASQUERADE which picks the src-ip based on NH from the route table
  183. } else {
  184. natArgs = []string{"-s", address, "!", "-o", bridgeIface, "-j", "MASQUERADE"}
  185. hpNatArgs = []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", bridgeIface, "-j", "MASQUERADE"}
  186. }
  187. natRule := iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: natArgs}
  188. hpNatRule := iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: hpNatArgs}
  189. ipVersion := iptables.IPv4
  190. if addr.IP.To4() == nil {
  191. ipVersion = iptables.IPv6
  192. }
  193. // Set NAT.
  194. if ipmasq {
  195. if err := programChainRule(ipVersion, natRule, "NAT", enable); err != nil {
  196. return err
  197. }
  198. }
  199. if ipmasq && !hairpin {
  200. if err := programChainRule(ipVersion, skipDNAT, "SKIP DNAT", enable); err != nil {
  201. return err
  202. }
  203. }
  204. // In hairpin mode, masquerade traffic from localhost
  205. if hairpin {
  206. if err := programChainRule(ipVersion, hpNatRule, "MASQ LOCAL HOST", enable); err != nil {
  207. return err
  208. }
  209. }
  210. // Set Inter Container Communication.
  211. if err := setIcc(ipVersion, bridgeIface, icc, enable); err != nil {
  212. return err
  213. }
  214. // Set Accept on all non-intercontainer outgoing packets.
  215. return programChainRule(ipVersion, outRule, "ACCEPT NON_ICC OUTGOING", enable)
  216. }
  217. func programChainRule(version iptables.IPVersion, rule iptRule, ruleDescr string, insert bool) error {
  218. iptable := iptables.GetIptable(version)
  219. var (
  220. prefix []string
  221. operation string
  222. condition bool
  223. doesExist = iptable.Exists(rule.table, rule.chain, rule.args...)
  224. )
  225. if insert {
  226. condition = !doesExist
  227. prefix = []string{"-I", rule.chain}
  228. operation = "enable"
  229. } else {
  230. condition = doesExist
  231. prefix = []string{"-D", rule.chain}
  232. operation = "disable"
  233. }
  234. if rule.preArgs != nil {
  235. prefix = append(rule.preArgs, prefix...)
  236. }
  237. if condition {
  238. if err := iptable.RawCombinedOutput(append(prefix, rule.args...)...); err != nil {
  239. return fmt.Errorf("Unable to %s %s rule: %s", operation, ruleDescr, err.Error())
  240. }
  241. }
  242. return nil
  243. }
  244. func setIcc(version iptables.IPVersion, bridgeIface string, iccEnable, insert bool) error {
  245. iptable := iptables.GetIptable(version)
  246. var (
  247. table = iptables.Filter
  248. chain = "FORWARD"
  249. args = []string{"-i", bridgeIface, "-o", bridgeIface, "-j"}
  250. acceptArgs = append(args, "ACCEPT")
  251. dropArgs = append(args, "DROP")
  252. )
  253. if insert {
  254. if !iccEnable {
  255. iptable.Raw(append([]string{"-D", chain}, acceptArgs...)...)
  256. if !iptable.Exists(table, chain, dropArgs...) {
  257. if err := iptable.RawCombinedOutput(append([]string{"-A", chain}, dropArgs...)...); err != nil {
  258. return fmt.Errorf("Unable to prevent intercontainer communication: %s", err.Error())
  259. }
  260. }
  261. } else {
  262. iptable.Raw(append([]string{"-D", chain}, dropArgs...)...)
  263. if !iptable.Exists(table, chain, acceptArgs...) {
  264. if err := iptable.RawCombinedOutput(append([]string{"-I", chain}, acceptArgs...)...); err != nil {
  265. return fmt.Errorf("Unable to allow intercontainer communication: %s", err.Error())
  266. }
  267. }
  268. }
  269. } else {
  270. // Remove any ICC rule.
  271. if !iccEnable {
  272. if iptable.Exists(table, chain, dropArgs...) {
  273. iptable.Raw(append([]string{"-D", chain}, dropArgs...)...)
  274. }
  275. } else {
  276. if iptable.Exists(table, chain, acceptArgs...) {
  277. iptable.Raw(append([]string{"-D", chain}, acceptArgs...)...)
  278. }
  279. }
  280. }
  281. return nil
  282. }
  283. // Control Inter Network Communication. Install[Remove] only if it is [not] present.
  284. func setINC(version iptables.IPVersion, iface string, enable bool) error {
  285. iptable := iptables.GetIptable(version)
  286. var (
  287. action = iptables.Insert
  288. actionMsg = "add"
  289. chains = []string{IsolationChain1, IsolationChain2}
  290. rules = [][]string{
  291. {"-i", iface, "!", "-o", iface, "-j", IsolationChain2},
  292. {"-o", iface, "-j", "DROP"},
  293. }
  294. )
  295. if !enable {
  296. action = iptables.Delete
  297. actionMsg = "remove"
  298. }
  299. for i, chain := range chains {
  300. if err := iptable.ProgramRule(iptables.Filter, chain, action, rules[i]); err != nil {
  301. msg := fmt.Sprintf("unable to %s inter-network communication rule: %v", actionMsg, err)
  302. if enable {
  303. if i == 1 {
  304. // Rollback the rule installed on first chain
  305. if err2 := iptable.ProgramRule(iptables.Filter, chains[0], iptables.Delete, rules[0]); err2 != nil {
  306. logrus.Warnf("Failed to rollback iptables rule after failure (%v): %v", err, err2)
  307. }
  308. }
  309. return fmt.Errorf(msg)
  310. }
  311. logrus.Warn(msg)
  312. }
  313. }
  314. return nil
  315. }
  316. // Obsolete chain from previous docker versions
  317. const oldIsolationChain = "DOCKER-ISOLATION"
  318. func removeIPChains(version iptables.IPVersion) {
  319. ipt := iptables.IPTable{Version: version}
  320. // Remove obsolete rules from default chains
  321. ipt.ProgramRule(iptables.Filter, "FORWARD", iptables.Delete, []string{"-j", oldIsolationChain})
  322. // Remove chains
  323. for _, chainInfo := range []iptables.ChainInfo{
  324. {Name: DockerChain, Table: iptables.Nat, IPTable: ipt},
  325. {Name: DockerChain, Table: iptables.Filter, IPTable: ipt},
  326. {Name: IsolationChain1, Table: iptables.Filter, IPTable: ipt},
  327. {Name: IsolationChain2, Table: iptables.Filter, IPTable: ipt},
  328. {Name: oldIsolationChain, Table: iptables.Filter, IPTable: ipt},
  329. } {
  330. if err := chainInfo.Remove(); err != nil {
  331. logrus.Warnf("Failed to remove existing iptables entries in table %s chain %s : %v", chainInfo.Table, chainInfo.Name, err)
  332. }
  333. }
  334. }
  335. func setupInternalNetworkRules(bridgeIface string, addr *net.IPNet, icc, insert bool) error {
  336. var (
  337. inDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{"-i", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}}
  338. outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{"-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}}
  339. )
  340. version := iptables.IPv4
  341. if addr.IP.To4() == nil {
  342. version = iptables.IPv6
  343. }
  344. if err := programChainRule(version, inDropRule, "DROP INCOMING", insert); err != nil {
  345. return err
  346. }
  347. if err := programChainRule(version, outDropRule, "DROP OUTGOING", insert); err != nil {
  348. return err
  349. }
  350. // Set Inter Container Communication.
  351. return setIcc(version, bridgeIface, icc, insert)
  352. }
  353. func clearEndpointConnections(nlh *netlink.Handle, ep *bridgeEndpoint) {
  354. var ipv4List []net.IP
  355. var ipv6List []net.IP
  356. if ep.addr != nil {
  357. ipv4List = append(ipv4List, ep.addr.IP)
  358. }
  359. if ep.addrv6 != nil {
  360. ipv6List = append(ipv6List, ep.addrv6.IP)
  361. }
  362. iptables.DeleteConntrackEntries(nlh, ipv4List, ipv6List)
  363. }