Embedded DNS server

Signed-off-by: Santhosh Manohar <santhosh@docker.com>

libnetwork/controller.go

@@ -143,7 +143,7 @@ type controller struct {
 	extKeyListener net.Listener
 	watchCh        chan *endpoint
 	unWatchCh      chan *endpoint
-	svcDb          map[string]svcMap
+	svcDb          map[string]svcInfo
 	nmap           map[string]*netWatch
 	defOsSbox      osl.Sandbox
 	sboxOnce       sync.Once
@@ -171,7 +171,7 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
 		sandboxes:   sandboxTable{},
 		drivers:     driverTable{},
 		ipamDrivers: ipamTable{},
-		svcDb:       make(map[string]svcMap),
+		svcDb:       make(map[string]svcInfo),
 	}
 
 	if err := c.initStores(); err != nil {

libnetwork/endpoint.go

@@ -50,21 +50,22 @@ type Endpoint interface {
 type EndpointOption func(ep *endpoint)
 
 type endpoint struct {
-	name          string
-	id            string
-	network       *network
-	iface         *endpointInterface
-	joinInfo      *endpointJoinInfo
-	sandboxID     string
-	exposedPorts  []types.TransportPort
-	anonymous     bool
-	generic       map[string]interface{}
-	joinLeaveDone chan struct{}
-	prefAddress   net.IP
-	prefAddressV6 net.IP
-	ipamOptions   map[string]string
-	dbIndex       uint64
-	dbExists      bool
+	name              string
+	id                string
+	network           *network
+	iface             *endpointInterface
+	joinInfo          *endpointJoinInfo
+	sandboxID         string
+	exposedPorts      []types.TransportPort
+	anonymous         bool
+	disableResolution bool
+	generic           map[string]interface{}
+	joinLeaveDone     chan struct{}
+	prefAddress       net.IP
+	prefAddressV6     net.IP
+	ipamOptions       map[string]string
+	dbIndex           uint64
+	dbExists          bool
 	sync.Mutex
 }
 
@@ -82,6 +83,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) {
 	}
 	epMap["sandbox"] = ep.sandboxID
 	epMap["anonymous"] = ep.anonymous
+	epMap["disableResolution"] = ep.disableResolution
 	return json.Marshal(epMap)
 }
 
@@ -159,6 +161,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
 	if v, ok := epMap["anonymous"]; ok {
 		ep.anonymous = v.(bool)
 	}
+	if v, ok := epMap["disableResolution"]; ok {
+		ep.disableResolution = v.(bool)
+	}
 	return nil
 }
 
@@ -177,6 +182,7 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error {
 	dstEp.dbIndex = ep.dbIndex
 	dstEp.dbExists = ep.dbExists
 	dstEp.anonymous = ep.anonymous
+	dstEp.disableResolution = ep.disableResolution
 
 	if ep.iface != nil {
 		dstEp.iface = &endpointInterface{}
@@ -222,6 +228,12 @@ func (ep *endpoint) isAnonymous() bool {
 	return ep.anonymous
 }
 
+func (ep *endpoint) needResolver() bool {
+	ep.Lock()
+	defer ep.Unlock()
+	return !ep.disableResolution
+}
+
 // endpoint Key structure : endpoint/network-id/endpoint-id
 func (ep *endpoint) Key() []string {
 	if ep.network == nil {
@@ -396,10 +408,9 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error {
 	if ip := ep.getFirstInterfaceAddress(); ip != nil {
 		address = ip.String()
 	}
-	if err = sb.updateHostsFile(address, network.getSvcRecords(ep)); err != nil {
+	if err = sb.updateHostsFile(address); err != nil {
 		return err
 	}
-
 	if err = sb.updateDNS(network.enableIPv6); err != nil {
 		return err
 	}
@@ -729,6 +740,14 @@ func CreateOptionAnonymous() EndpointOption {
 	}
 }
 
+// CreateOptionDisableResolution function returns an option setter to indicate
+// this endpoint doesn't want embedded DNS server functionality
+func CreateOptionDisableResolution() EndpointOption {
+	return func(ep *endpoint) {
+		ep.disableResolution = true
+	}
+}
+
 // JoinOptionPriority function returns an option setter for priority option to
 // be passed to the endpoint.Join() method.
 func JoinOptionPriority(ep Endpoint, prio int) EndpointOption {

libnetwork/libnetwork_test.go

@@ -1213,6 +1213,14 @@ func (f *fakeSandbox) SetKey(key string) error {
 	return nil
 }
 
+func (f *fakeSandbox) ResolveName(name string) net.IP {
+	return nil
+}
+
+func (f *fakeSandbox) ResolveIP(ip string) string {
+	return ""
+}
+
 func TestExternalKey(t *testing.T) {
 	externalKeyTest(t, false)
 }
@@ -1698,6 +1706,7 @@ func TestEnableIPv6(t *testing.T) {
 	}
 
 	tmpResolvConf := []byte("search pommesfrites.fr\nnameserver 12.34.56.78\nnameserver 2001:4860:4860::8888\n")
+	expectedResolvConf := []byte("search pommesfrites.fr\nnameserver 127.0.0.11\noptions ndots:0\n")
 	//take a copy of resolv.conf for restoring after test completes
 	resolvConfSystem, err := ioutil.ReadFile("/etc/resolv.conf")
 	if err != nil {
@@ -1760,8 +1769,8 @@ func TestEnableIPv6(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if !bytes.Equal(content, tmpResolvConf) {
-		t.Fatalf("Expected:\n%s\nGot:\n%s", string(tmpResolvConf), string(content))
+	if !bytes.Equal(content, expectedResolvConf) {
+		t.Fatalf("Expected:\n%s\nGot:\n%s", string(expectedResolvConf), string(content))
 	}
 
 	if err != nil {
@@ -1793,7 +1802,7 @@ func TestResolvConfHost(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	ep1, err := n.CreateEndpoint("ep1", nil)
+	ep1, err := n.CreateEndpoint("ep1", libnetwork.CreateOptionDisableResolution())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1854,9 +1863,8 @@ func TestResolvConf(t *testing.T) {
 	}
 
 	tmpResolvConf1 := []byte("search pommesfrites.fr\nnameserver 12.34.56.78\nnameserver 2001:4860:4860::8888\n")
-	expectedResolvConf1 := []byte("search pommesfrites.fr\nnameserver 12.34.56.78\n")
 	tmpResolvConf2 := []byte("search pommesfrites.fr\nnameserver 112.34.56.78\nnameserver 2001:4860:4860::8888\n")
-	expectedResolvConf2 := []byte("search pommesfrites.fr\nnameserver 112.34.56.78\n")
+	expectedResolvConf1 := []byte("search pommesfrites.fr\nnameserver 127.0.0.11\noptions ndots:0\n")
 	tmpResolvConf3 := []byte("search pommesfrites.fr\nnameserver 113.34.56.78\n")
 
 	//take a copy of resolv.conf for restoring after test completes
@@ -1965,8 +1973,8 @@ func TestResolvConf(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if !bytes.Equal(content, expectedResolvConf2) {
-		t.Fatalf("Expected:\n%s\nGot:\n%s", string(expectedResolvConf2), string(content))
+	if !bytes.Equal(content, expectedResolvConf1) {
+		t.Fatalf("Expected:\n%s\nGot:\n%s", string(expectedResolvConf1), string(content))
 	}
 
 	if err := ioutil.WriteFile(resolvConfPath, tmpResolvConf3, 0644); err != nil {

libnetwork/netutils/utils.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"strings"
 
 	"github.com/docker/libnetwork/types"
 )
@@ -132,3 +133,39 @@ func GenerateRandomName(prefix string, size int) (string, error) {
 	}
 	return prefix + hex.EncodeToString(id)[:size], nil
 }
+
+// ReverseIP accepts a V4 or V6 IP string in the canonical form and returns a reversed IP in
+// the dotted decimal form . This is used to setup the IP to service name mapping in the optimal
+// way for the DNS PTR queries.
+func ReverseIP(IP string) string {
+	var reverseIP []string
+
+	if net.ParseIP(IP).To4() != nil {
+		reverseIP = strings.Split(IP, ".")
+		l := len(reverseIP)
+		for i, j := 0, l-1; i < l/2; i, j = i+1, j-1 {
+			reverseIP[i], reverseIP[j] = reverseIP[j], reverseIP[i]
+		}
+	} else {
+		reverseIP = strings.Split(IP, ":")
+
+		// Reversed IPv6 is represented in dotted decimal instead of the typical
+		// colon hex notation
+		for key := range reverseIP {
+			if len(reverseIP[key]) == 0 { // expand the compressed 0s
+				reverseIP[key] = strings.Repeat("0000", 8-strings.Count(IP, ":"))
+			} else if len(reverseIP[key]) < 4 { // 0-padding needed
+				reverseIP[key] = strings.Repeat("0", 4-len(reverseIP[key])) + reverseIP[key]
+			}
+		}
+
+		reverseIP = strings.Split(strings.Join(reverseIP, ""), "")
+
+		l := len(reverseIP)
+		for i, j := 0, l-1; i < l/2; i, j = i+1, j-1 {
+			reverseIP[i], reverseIP[j] = reverseIP[j], reverseIP[i]
+		}
+	}
+
+	return strings.Join(reverseIP, ".")
+}

libnetwork/network.go

@@ -69,7 +69,10 @@ type NetworkInfo interface {
 // When the function returns true, the walk will stop.
 type EndpointWalker func(ep Endpoint) bool
 
-type svcMap map[string]net.IP
+type svcInfo struct {
+	svcMap map[string]net.IP
+	ipMap  map[string]string
+}
 
 // IpamConf contains all the ipam related configurations for a network
 type IpamConf struct {
@@ -159,7 +162,6 @@ type network struct {
 	epCnt        *endpointCnt
 	generic      options.Generic
 	dbIndex      uint64
-	svcRecords   svcMap
 	dbExists     bool
 	persist      bool
 	stopWatchCh  chan struct{}
@@ -832,62 +834,33 @@ func (n *network) updateSvcRecord(ep *endpoint, localEps []*endpoint, isAdd bool
 	c := n.getController()
 	sr, ok := c.svcDb[n.ID()]
 	if !ok {
-		c.svcDb[n.ID()] = svcMap{}
+		c.svcDb[n.ID()] = svcInfo{
+			svcMap: make(map[string]net.IP),
+			ipMap:  make(map[string]string),
+		}
 		sr = c.svcDb[n.ID()]
 	}
 
+	epName := ep.Name()
 	n.Lock()
-	var recs []etchosts.Record
 	if iface := ep.Iface(); iface.Address() != nil {
+
+		reverseIP := netutils.ReverseIP(iface.Address().IP.String())
 		if isAdd {
 			// If we already have this endpoint in service db just return
-			if _, ok := sr[ep.Name()]; ok {
+			if _, ok := sr.svcMap[epName]; ok {
 				n.Unlock()
 				return
 			}
 
-			sr[ep.Name()] = iface.Address().IP
-			sr[ep.Name()+"."+n.name] = iface.Address().IP
+			sr.svcMap[epName] = iface.Address().IP
+			sr.ipMap[reverseIP] = epName
 		} else {
-			delete(sr, ep.Name())
-			delete(sr, ep.Name()+"."+n.name)
+			delete(sr.svcMap, epName)
+			delete(sr.ipMap, reverseIP)
 		}
-
-		recs = append(recs, etchosts.Record{
-			Hosts: ep.Name(),
-			IP:    iface.Address().IP.String(),
-		})
-
-		recs = append(recs, etchosts.Record{
-			Hosts: ep.Name() + "." + n.name,
-			IP:    iface.Address().IP.String(),
-		})
 	}
 	n.Unlock()
-
-	// If there are no records to add or delete then simply return here
-	if len(recs) == 0 {
-		return
-	}
-
-	var sbList []*sandbox
-	for _, lEp := range localEps {
-		if ep.ID() == lEp.ID() {
-			continue
-		}
-
-		if sb, hasSandbox := lEp.getSandbox(); hasSandbox {
-			sbList = append(sbList, sb)
-		}
-	}
-
-	for _, sb := range sbList {
-		if isAdd {
-			sb.addHostsEntries(recs)
-		} else {
-			sb.deleteHostsEntries(recs)
-		}
-	}
 }
 
 func (n *network) getSvcRecords(ep *endpoint) []etchosts.Record {
@@ -897,7 +870,7 @@ func (n *network) getSvcRecords(ep *endpoint) []etchosts.Record {
 	var recs []etchosts.Record
 	sr, _ := n.ctrlr.svcDb[n.id]
 
-	for h, ip := range sr {
+	for h, ip := range sr.svcMap {
 		if ep != nil && strings.Split(h, ".")[0] == ep.Name() {
 			continue
 		}

libnetwork/resolver.go

@@ -0,0 +1,205 @@
+package libnetwork
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/docker/libnetwork/iptables"
+	"github.com/miekg/dns"
+)
+
+// Resolver represents the embedded DNS server in Docker. It operates
+// by listening on container's loopback interface for DNS queries.
+type Resolver interface {
+	// Start starts the name server for the container
+	Start() error
+	// Stop stops the name server for the container
+	Stop()
+	// SetupFunc() provides the setup function that should be run
+	// in the container's network namespace.
+	SetupFunc() func()
+	// NameServer() returns the IP of the DNS resolver for the
+	// containers.
+	NameServer() string
+	// To configure external name servers the resolver should use
+	SetExtServers([]string)
+	// ResolverOptions returns resolv.conf options that should be set
+	ResolverOptions() []string
+}
+
+const (
+	resolverIP    = "127.0.0.11"
+	dnsPort       = "53"
+	ptrIPv4domain = ".in-addr.arpa."
+	ptrIPv6domain = ".ip6.arpa."
+	respTTL       = 1800
+)
+
+// resolver implements the Resolver interface
+type resolver struct {
+	sb     *sandbox
+	extDNS []string
+	server *dns.Server
+	conn   *net.UDPConn
+	err    error
+}
+
+// NewResolver creates a new instance of the Resolver
+func NewResolver(sb *sandbox) Resolver {
+	return &resolver{
+		sb:  sb,
+		err: fmt.Errorf("setup not done yet"),
+	}
+}
+
+func (r *resolver) SetupFunc() func() {
+	return (func() {
+		var err error
+
+		addr := &net.UDPAddr{
+			IP: net.ParseIP(resolverIP),
+		}
+
+		r.conn, err = net.ListenUDP("udp", addr)
+		if err != nil {
+			r.err = fmt.Errorf("error in opening name server socket %v", err)
+			return
+		}
+		laddr := r.conn.LocalAddr()
+		_, ipPort, _ := net.SplitHostPort(laddr.String())
+
+		rules := [][]string{
+			{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", laddr.String()},
+			{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
+		}
+
+		for _, rule := range rules {
+			r.err = iptables.RawCombinedOutput(rule...)
+			if r.err != nil {
+				return
+			}
+		}
+		r.err = nil
+	})
+}
+
+func (r *resolver) Start() error {
+	// make sure the resolver has been setup before starting
+	if r.err != nil {
+		return r.err
+	}
+	s := &dns.Server{Handler: r, PacketConn: r.conn}
+	r.server = s
+	go func() {
+		s.ActivateAndServe()
+	}()
+	return nil
+}
+
+func (r *resolver) Stop() {
+	if r.server != nil {
+		r.server.Shutdown()
+	}
+}
+
+func (r *resolver) SetExtServers(dns []string) {
+	r.extDNS = dns
+}
+
+func (r *resolver) NameServer() string {
+	return resolverIP
+}
+
+func (r *resolver) ResolverOptions() []string {
+	return []string{"ndots:0"}
+}
+
+func (r *resolver) handleIPv4Query(name string, query *dns.Msg) (*dns.Msg, error) {
+	addr := r.sb.ResolveName(name)
+	if addr == nil {
+		return nil, nil
+	}
+
+	log.Debugf("Lookup for %s: IP %s", name, addr.String())
+
+	resp := new(dns.Msg)
+	resp.SetReply(query)
+
+	rr := new(dns.A)
+	rr.Hdr = dns.RR_Header{Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: respTTL}
+	rr.A = addr
+	resp.Answer = append(resp.Answer, rr)
+	return resp, nil
+}
+
+func (r *resolver) handlePTRQuery(ptr string, query *dns.Msg) (*dns.Msg, error) {
+	parts := []string{}
+
+	if strings.HasSuffix(ptr, ptrIPv4domain) {
+		parts = strings.Split(ptr, ptrIPv4domain)
+	} else if strings.HasSuffix(ptr, ptrIPv6domain) {
+		parts = strings.Split(ptr, ptrIPv6domain)
+	} else {
+		return nil, fmt.Errorf("invalid PTR query, %v", ptr)
+	}
+
+	host := r.sb.ResolveIP(parts[0])
+	if len(host) == 0 {
+		return nil, nil
+	}
+
+	log.Debugf("Lookup for IP %s: name %s", parts[0], host)
+	fqdn := dns.Fqdn(host)
+
+	resp := new(dns.Msg)
+	resp.SetReply(query)
+
+	rr := new(dns.PTR)
+	rr.Hdr = dns.RR_Header{Name: ptr, Rrtype: dns.TypePTR, Class: dns.ClassINET, Ttl: respTTL}
+	rr.Ptr = fqdn
+	resp.Answer = append(resp.Answer, rr)
+	return resp, nil
+}
+
+func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {
+	var (
+		resp *dns.Msg
+		err  error
+	)
+
+	name := query.Question[0].Name
+	if query.Question[0].Qtype == dns.TypeA {
+		resp, err = r.handleIPv4Query(name, query)
+	} else if query.Question[0].Qtype == dns.TypePTR {
+		resp, err = r.handlePTRQuery(name, query)
+	}
+
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	if resp == nil {
+		if len(r.extDNS) == 0 {
+			return
+		}
+		log.Debugf("Querying ext dns %s for %s[%d]", r.extDNS[0], name, query.Question[0].Qtype)
+
+		c := &dns.Client{Net: "udp"}
+		addr := fmt.Sprintf("%s:%d", r.extDNS[0], 53)
+
+		// TODO: iterate over avilable servers in case of error
+		resp, _, err = c.Exchange(query, addr)
+		if err != nil {
+			log.Errorf("external resolution failed, %s", err)
+			return
+		}
+	}
+
+	err = w.WriteMsg(resp)
+	if err != nil {
+		log.Errorf("error writing resolver resp, %s", err)
+	}
+}

libnetwork/sandbox.go

@@ -5,9 +5,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	log "github.com/Sirupsen/logrus"
@@ -38,6 +40,12 @@ type Sandbox interface {
 	Rename(name string) error
 	// Delete destroys this container after detaching it from all connected endpoints.
 	Delete() error
+	// ResolveName searches for the service name in the networks to which the sandbox
+	// is connected to.
+	ResolveName(name string) net.IP
+	// ResolveIP returns the service name for the passed in IP. IP is in reverse dotted
+	// notation; the format used for DNS PTR records
+	ResolveIP(name string) string
 }
 
 // SandboxOption is a option setter function type used to pass varios options to
@@ -59,8 +67,11 @@ type sandbox struct {
 	id            string
 	containerID   string
 	config        containerConfig
+	extDNS        []string
 	osSbox        osl.Sandbox
 	controller    *controller
+	resolver      Resolver
+	resolverOnce  sync.Once
 	refCnt        int
 	endpoints     epHeap
 	epPriority    map[string]int
@@ -202,6 +213,10 @@ func (sb *sandbox) Delete() error {
 	// likely not required any more. Drop it.
 	etchosts.Drop(sb.config.hostsPath)
 
+	if sb.resolver != nil {
+		sb.resolver.Stop()
+	}
+
 	if sb.osSbox != nil && !sb.config.useDefaultSandBox {
 		sb.osSbox.Destroy()
 	}
@@ -291,6 +306,26 @@ func (sb *sandbox) UnmarshalJSON(b []byte) (err error) {
 	return nil
 }
 
+func (sb *sandbox) startResolver() {
+	sb.resolverOnce.Do(func() {
+		var err error
+		sb.resolver = NewResolver(sb)
+		defer func() {
+			if err != nil {
+				sb.resolver = nil
+			}
+		}()
+
+		sb.rebuildDNS()
+		sb.resolver.SetExtServers(sb.extDNS)
+
+		sb.osSbox.InvokeFunc(sb.resolver.SetupFunc())
+		if err := sb.resolver.Start(); err != nil {
+			log.Errorf("Resolver Setup/Start failed for container %s, %q", sb.ContainerID(), err)
+		}
+	})
+}
+
 func (sb *sandbox) setupResolutionFiles() error {
 	if err := sb.buildHostsFile(); err != nil {
 		return err
@@ -361,6 +396,56 @@ func (sb *sandbox) updateGateway(ep *endpoint) error {
 	return nil
 }
 
+func (sb *sandbox) ResolveIP(ip string) string {
+	var svc string
+	log.Debugf("IP To resolve %v", ip)
+
+	for _, ep := range sb.getConnectedEndpoints() {
+		n := ep.getNetwork()
+
+		sr, ok := n.getController().svcDb[n.ID()]
+		if !ok {
+			continue
+		}
+
+		nwName := n.Name()
+		n.Lock()
+		svc, ok = sr.ipMap[ip]
+		n.Unlock()
+		if ok {
+			return svc + "." + nwName
+		}
+	}
+	return svc
+}
+
+func (sb *sandbox) ResolveName(name string) net.IP {
+	var ip net.IP
+	parts := strings.Split(name, ".")
+	log.Debugf("To resolve %v", parts)
+
+	for _, ep := range sb.getConnectedEndpoints() {
+		n := ep.getNetwork()
+
+		if len(parts) > 1 && parts[1] != "" && parts[1] != n.Name() {
+			continue
+		}
+
+		sr, ok := n.getController().svcDb[n.ID()]
+		if !ok {
+			continue
+		}
+
+		n.Lock()
+		ip, ok = sr.svcMap[parts[0]]
+		n.Unlock()
+		if ok {
+			return ip
+		}
+	}
+	return ip
+}
+
 func (sb *sandbox) SetKey(basePath string) error {
 	var err error
 	if basePath == "" {
@@ -459,6 +544,10 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
 	i := ep.iface
 	ep.Unlock()
 
+	if ep.needResolver() {
+		sb.startResolver()
+	}
+
 	if i != nil && i.srcName != "" {
 		var ifaceOptions []osl.IfaceOption
 
@@ -599,7 +688,7 @@ func (sb *sandbox) buildHostsFile() error {
 	return etchosts.Build(sb.config.hostsPath, "", sb.config.hostName, sb.config.domainName, extraContent)
 }
 
-func (sb *sandbox) updateHostsFile(ifaceIP string, svcRecords []etchosts.Record) error {
+func (sb *sandbox) updateHostsFile(ifaceIP string) error {
 	var mhost string
 
 	if sb.config.originHostsPath != "" {
@@ -613,11 +702,7 @@ func (sb *sandbox) updateHostsFile(ifaceIP string, svcRecords []etchosts.Record)
 		mhost = sb.config.hostName
 	}
 
-	extraContent := make([]etchosts.Record, 0, len(svcRecords)+1)
-	extraContent = append(extraContent, etchosts.Record{Hosts: mhost, IP: ifaceIP})
-	for _, svc := range svcRecords {
-		extraContent = append(extraContent, svc)
-	}
+	extraContent := []etchosts.Record{{Hosts: mhost, IP: ifaceIP}}
 
 	sb.addHostsEntries(extraContent)
 	return nil
@@ -787,6 +872,48 @@ func (sb *sandbox) updateDNS(ipv6Enabled bool) error {
 	return os.Rename(tmpResolvFile.Name(), sb.config.resolvConfPath)
 }
 
+// Embedded DNS server has to be enabled for this sandbox. Rebuild the container's
+// resolv.conf by doing the follwing
+// - Save the external name servers in resolv.conf in the sandbox
+// - Add only the embedded server's IP to container's resolv.conf
+// - If the embedded server needs any resolv.conf options add it to the current list
+func (sb *sandbox) rebuildDNS() error {
+	currRC, err := resolvconf.GetSpecific(sb.config.resolvConfPath)
+	if err != nil {
+		return err
+	}
+
+	// localhost entries have already been filtered out from the list
+	sb.extDNS = resolvconf.GetNameservers(currRC.Content)
+
+	var (
+		dnsList        = []string{sb.resolver.NameServer()}
+		dnsOptionsList = resolvconf.GetOptions(currRC.Content)
+		dnsSearchList  = resolvconf.GetSearchDomains(currRC.Content)
+	)
+
+	// Resolver returns the options in the format resolv.conf expects
+	dnsOptionsList = append(dnsOptionsList, sb.resolver.ResolverOptions()...)
+
+	dir := path.Dir(sb.config.resolvConfPath)
+	tmpResolvFile, err := ioutil.TempFile(dir, "resolv")
+	if err != nil {
+		return err
+	}
+
+	// Change the perms to filePerm (0644) since ioutil.TempFile creates it by default as 0600
+	if err := os.Chmod(tmpResolvFile.Name(), filePerm); err != nil {
+		return err
+	}
+
+	_, err = resolvconf.Build(tmpResolvFile.Name(), dnsList, dnsSearchList, dnsOptionsList)
+	if err != nil {
+		return err
+	}
+
+	return os.Rename(tmpResolvFile.Name(), sb.config.resolvConfPath)
+}
+
 // joinLeaveStart waits to ensure there are no joins or leaves in progress and
 // marks this join/leave in progress without race
 func (sb *sandbox) joinLeaveStart() {

libnetwork/test/integration/dnet/bridge.bats

@@ -179,6 +179,23 @@ function test_single_network_connectivity() {
 	done
     done
 
+    svcs=(
+	0,0
+	2,3
+	1,3
+	1,2
+    )
+
+    echo "Test connectivity failure"
+    for i in `seq ${start} ${end}`;
+    do
+	IFS=, read a b <<<"${svcs[$i]}"
+	osvc="svc${a}${b}"
+	echo "pinging ${osvc}"
+	runc_nofail $(dnet_container_name 1 bridge) $(get_sbox_id 1 container_${i}) "ping -c 1 ${osvc}"
+	[ "${status}" -ne 0 ]
+    done
+
     for i in `seq ${start} ${end}`;
     do
 	for j in `seq ${start} ${end}`;

libnetwork/test/integration/dnet/helpers.bash

@@ -351,84 +351,6 @@ function check_etchosts() {
     echo ${retval}
 }
 
-function test_overlay_etchosts() {
-    local clist dnet_suffix
-
-    dnet_suffix=$1
-    shift
-
-    echo $(docker ps)
-
-    start=1
-    end=3
-    # Setup overlay network and connect containers ot it
-    dnet_cmd $(inst_id2port 1) network create -d overlay multihost
-
-    for iter in `seq 1 2`;
-    do
-	for i in `seq ${start} ${end}`;
-	do
-	    dnet_cmd $(inst_id2port $i) container create container_${iter}_${i}
-	    net_connect ${i} container_${iter}_${i} multihost
-	done
-
-	# Now test the /etc/hosts content of all the containers
-	for i in `seq ${start} ${end}`;
-	do
-	    clist=""
-	    oldclist=""
-	    for j in `seq ${start} ${end}`;
-	    do
-		if [ "$i" -eq "$j" ]; then
-		    continue
-		fi
-		clist="$clist container_${iter}_$j"
-		oldclist="$oldclist container_1_$j"
-	    done
-	    rv=$(check_etchosts $(dnet_container_name $i $dnet_suffix) \
-				$(get_sbox_id ${i} container_${iter}_${i}) \
-				${clist})
-	    [ "$rv" = "true" ]
-
-	    # check to see the containers don't have stale entries from previous iteration
-	    if [ "$iter" -eq 2 ]; then
-		rv=$(check_etchosts $(dnet_container_name $i $dnet_suffix) \
-				    $(get_sbox_id ${i} container_${iter}_${i}) \
-				    ${oldclist})
-		[ "$rv" = "false" ]
-	    fi
-	done
-
-	# Teardown the container connections and the network
-	clist=""
-	for i in `seq ${start} ${end}`;
-	do
-	    net_disconnect ${i} container_${iter}_${i} multihost
-	    dnet_cmd $(inst_id2port $i) container rm container_${iter}_${i}
-
-	    #check if the /etc/hosts of other containers does not contain this container
-	    for j in `seq ${start} ${end}`;
-	    do
-		if [ "$i" -eq "$j" ]; then
-		    continue
-		fi
-
-		if [[ "${clist}" =~ .*container_${iter}_${j}.* ]]; then
-		    continue
-		fi
-
-		rv=$(check_etchosts $(dnet_container_name $j $dnet_suffix) \
-				    $(get_sbox_id ${j} container_${iter}_${j}) \
-				    container_${iter}_${i})
-		[ "$rv" = "false" ]
-	    done
-	    clist="${clist} container_${iter}_${i}"
-	done
-    done
-
-    dnet_cmd $(inst_id2port 2) network rm multihost
-}
-
 function test_overlay_singlehost() {
     dnet_suffix=$1
     shift

libnetwork/test/integration/dnet/overlay-consul.bats

@@ -13,11 +13,6 @@ load helpers
     test_overlay_singlehost consul
 }
 
-@test "test overlay network etc hosts with consul" {
-    skip_for_circleci
-    test_overlay_etchosts consul
-}
-
 @test "Test overlay network with dnet restart" {
     skip_for_circleci
     test_overlay consul skip_rm
@@ -33,4 +28,4 @@ load helpers
 @test "Test overlay network internal network with consul" {
     skip_for_circleci
     test_overlay consul internal
-}
+}