libn/d/overlay: enforce encryption on sandbox init

The iptables rules which make encryption mandatory on an encrypted
overlay network are only programmed once there is a second node
participating in the network. This leaves single-node encrypted overlay
networks vulnerable to packet injection. Furthermore, failure to program
the rules is not treated as a fatal error.

Program the iptables rules to make encryption mandatory before creating
the VXLAN link to guarantee that there is no window of time where
incoming cleartext VXLAN packets for the network would be accepted, or
outgoing cleartext packets be transmitted. Only create the VXLAN link if
programming the rules succeeds to ensure that it fails closed.

Signed-off-by: Cory Snider <csnider@mirantis.com>

libnetwork/drivers/overlay/encryption.go

@@ -112,8 +112,8 @@ func (e *encrMap) String() string {
 	return b.String()
 }
 
-func (d *driver) checkEncryption(nid string, rIP net.IP, vxlanID uint32, isLocal, add bool) error {
-	logrus.Debugf("checkEncryption(%.7s, %v, %d, %t)", nid, rIP, vxlanID, isLocal)
+func (d *driver) checkEncryption(nid string, rIP net.IP, isLocal, add bool) error {
+	logrus.Debugf("checkEncryption(%.7s, %v, %t)", nid, rIP, isLocal)
 
 	n := d.network(nid)
 	if n == nil || !n.secure {
@@ -148,7 +148,7 @@ func (d *driver) checkEncryption(nid string, rIP net.IP, vxlanID uint32, isLocal
 
 	if add {
 		for _, rIP := range nodes {
-			if err := setupEncryption(lIP, aIP, rIP, vxlanID, d.secMap, d.keys); err != nil {
+			if err := setupEncryption(lIP, aIP, rIP, d.secMap, d.keys); err != nil {
 				logrus.Warnf("Failed to program network encryption between %s and %s: %v", lIP, rIP, err)
 			}
 		}
@@ -163,22 +163,14 @@ func (d *driver) checkEncryption(nid string, rIP net.IP, vxlanID uint32, isLocal
 	return nil
 }
 
-func setupEncryption(localIP, advIP, remoteIP net.IP, vni uint32, em *encrMap, keys []*key) error {
-	logrus.Debugf("Programming encryption for vxlan %d between %s and %s", vni, localIP, remoteIP)
+// setupEncryption programs the encryption parameters for secure communication
+// between the local node and a remote node.
+func setupEncryption(localIP, advIP, remoteIP net.IP, em *encrMap, keys []*key) error {
+	logrus.Debugf("Programming encryption between %s and %s", localIP, remoteIP)
 	rIPs := remoteIP.String()
 
 	indices := make([]*spi, 0, len(keys))
 
-	err := programMangle(vni, true)
-	if err != nil {
-		logrus.Warn(err)
-	}
-
-	err = programInput(vni, true)
-	if err != nil {
-		logrus.Warn(err)
-	}
-
 	for i, k := range keys {
 		spis := &spi{buildSPI(advIP, remoteIP, k.tag), buildSPI(remoteIP, advIP, k.tag)}
 		dir := reverse
@@ -233,37 +225,33 @@ func removeEncryption(localIP, remoteIP net.IP, em *encrMap) error {
 	return nil
 }
 
-func programMangle(vni uint32, add bool) (err error) {
+func programMangle(vni uint32, add bool) error {
 	var (
 		p      = strconv.FormatUint(uint64(overlayutils.VXLANUDPPort()), 10)
 		c      = fmt.Sprintf("0>>22&0x3C@12&0xFFFFFF00=%d", int(vni)<<8)
 		m      = strconv.FormatUint(mark, 10)
 		chain  = "OUTPUT"
 		rule   = []string{"-p", "udp", "--dport", p, "-m", "u32", "--u32", c, "-j", "MARK", "--set-mark", m}
-		a      = "-A"
+		a      = iptables.Append
 		action = "install"
 	)
 
 	// TODO IPv6 support
 	iptable := iptables.GetIptable(iptables.IPv4)
 
-	if add == iptable.Exists(iptables.Mangle, chain, rule...) {
-		return
-	}
-
 	if !add {
-		a = "-D"
+		a = iptables.Delete
 		action = "remove"
 	}
 
-	if err = iptable.RawCombinedOutput(append([]string{"-t", string(iptables.Mangle), a, chain}, rule...)...); err != nil {
-		logrus.Warnf("could not %s mangle rule: %v", action, err)
+	if err := iptable.ProgramRule(iptables.Mangle, chain, a, rule); err != nil {
+		return fmt.Errorf("could not %s mangle rule: %w", action, err)
 	}
 
-	return
+	return nil
 }
 
-func programInput(vni uint32, add bool) (err error) {
+func programInput(vni uint32, add bool) error {
 	var (
 		port       = strconv.FormatUint(uint64(overlayutils.VXLANUDPPort()), 10)
 		vniMatch   = fmt.Sprintf("0>>22&0x3C@12&0xFFFFFF00=%d", int(vni)<<8)
@@ -286,15 +274,15 @@ func programInput(vni uint32, add bool) (err error) {
 
 	// Accept incoming VXLAN datagrams for the VNI which were subjected to IPSec processing.
 	if err := iptable.ProgramRule(iptables.Filter, chain, action, accept); err != nil {
-		logrus.Errorf("could not %s input rule: %v. Please do it manually.", msg, err)
+		return fmt.Errorf("could not %s input accept rule: %w", msg, err)
 	}
 
 	// Drop incoming VXLAN datagrams for the VNI which were received in cleartext.
 	if err := iptable.ProgramRule(iptables.Filter, chain, action, block); err != nil {
-		logrus.Errorf("could not %s input rule: %v. Please do it manually.", msg, err)
+		return fmt.Errorf("could not %s input drop rule: %w", msg, err)
 	}
 
-	return
+	return nil
 }
 
 func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (fSA *netlink.XfrmState, rSA *netlink.XfrmState, err error) {

libnetwork/drivers/overlay/joinleave.go

@@ -117,7 +117,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 
 	d.peerAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), false, false, true)
 
-	if err = d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil {
+	if err = d.checkEncryption(nid, nil, true, true); err != nil {
 		logrus.Warn(err)
 	}
 

libnetwork/drivers/overlay/ov_network.go

@@ -22,6 +22,7 @@ import (
 	"github.com/docker/docker/libnetwork/osl"
 	"github.com/docker/docker/libnetwork/resolvconf"
 	"github.com/docker/docker/libnetwork/types"
+	"github.com/hashicorp/go-multierror"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netlink/nl"
@@ -654,6 +655,18 @@ func (n *network) initSubnetSandbox(s *subnet, restore bool) error {
 	brName := n.generateBridgeName(s)
 	vxlanName := n.generateVxlanName(s)
 
+	// Program iptables rules for mandatory encryption of the secure
+	// network, or clean up leftover rules for a stale secure network which
+	// was previously assigned the same VNI.
+	if err := programMangle(s.vni, n.secure); err != nil {
+		return err
+	}
+	if err := programInput(s.vni, n.secure); err != nil {
+		if n.secure {
+			return multierror.Append(err, programMangle(s.vni, false))
+		}
+	}
+
 	if restore {
 		if err := n.restoreSubnetSandbox(s, brName, vxlanName); err != nil {
 			return err

libnetwork/drivers/overlay/peerdb.go

@@ -387,7 +387,7 @@ func (d *driver) peerAddOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask
 		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
 	}
 
-	if err := d.checkEncryption(nid, vtep, n.vxlanID(s), false, true); err != nil {
+	if err := d.checkEncryption(nid, vtep, false, true); err != nil {
 		logrus.Warn(err)
 	}
 
@@ -447,7 +447,7 @@ func (d *driver) peerDeleteOp(nid, eid string, peerIP net.IP, peerIPMask net.IPM
 		return nil
 	}
 
-	if err := d.checkEncryption(nid, vtep, 0, localPeer, false); err != nil {
+	if err := d.checkEncryption(nid, vtep, localPeer, false); err != nil {
 		logrus.Warn(err)
 	}