Internal resolver for default bridge network

Until now, containers on the default bridge network have been configured
to talk directly to external DNS servers - their resolv.conf files have
either been populated with nameservers from the host's resolv.conf, or
with servers from '--dns' (or with Google's nameservers as a fallback).

This change makes the internal bridge more like other networks by using
the internal resolver.  But, the internal resolver is not populated with
container names or aliases - it's only for external DNS lookups.

Containers on the default network, on a host that has a loopback
resolver (like systemd's on 127.0.0.53) will now use that resolver
via the internal resolver. So, the logic used to find systemd's current
set of resolvers is no longer needed by the daemon.

Legacy links work just as they did before, using '/etc/hosts' and magic.

(Buildkit does not use libnetwork, so it can't use the internal resolver.
But it does use libnetwork/resolvconf's logic to configure resolv.conf.
So, code to set up resolv.conf for a legacy networking without an internal
resolver can't be removed yet.)

Signed-off-by: Rob Murray <rob.murray@docker.com>
This commit is contained in:
Rob Murray 2024-03-20 17:44:19 +00:00
parent 641e341eed
commit 6924f2c066
12 changed files with 78 additions and 215 deletions

View file

@ -87,9 +87,7 @@ type Config struct {
NoNewPrivileges bool `json:"no-new-privileges,omitempty"`
IpcMode string `json:"default-ipc-mode,omitempty"`
CgroupNamespaceMode string `json:"default-cgroupns-mode,omitempty"`
// ResolvConf is the path to the configuration of the host resolver
ResolvConf string `json:"resolv-conf,omitempty"`
Rootless bool `json:"rootless,omitempty"`
Rootless bool `json:"rootless,omitempty"`
}
// GetExecRoot returns the user configured Exec-root
@ -136,12 +134,6 @@ func (conf *Config) LookupInitPath() (string, error) {
return exec.LookPath(binary)
}
// GetResolvConf returns the appropriate resolv.conf
// Check setupResolvConf on how this is selected
func (conf *Config) GetResolvConf() string {
return conf.ResolvConf
}
// IsSwarmCompatible defines if swarm mode can be enabled in this config
func (conf *Config) IsSwarmCompatible() error {
if conf.LiveRestoreEnabled {

View file

@ -419,50 +419,20 @@ func serviceDiscoveryOnDefaultNetwork() bool {
func setupPathsAndSandboxOptions(container *container.Container, cfg *config.Config, sboxOptions *[]libnetwork.SandboxOption) error {
var err error
var originResolvConfPath string
// Set the correct paths for /etc/hosts and /etc/resolv.conf, based on the
// networking-mode of the container. Note that containers with "container"
// networking are already handled in "initializeNetworking()" before we reach
// this function, so do not have to be accounted for here.
switch {
case container.HostConfig.NetworkMode.IsHost():
// In host-mode networking, the container does not have its own networking
// namespace, so both `/etc/hosts` and `/etc/resolv.conf` should be the same
// as on the host itself. The container gets a copy of these files.
// In host-mode networking, the container does not have its own networking
// namespace, so `/etc/hosts` should be the same as on the host itself. Setting
// OptionOriginHostsPath means the container will get a copy of the host's file.
// Note that containers with "container" networking have been handled in
// "initializeNetworking()", so do not have to be accounted for here.
if container.HostConfig.NetworkMode.IsHost() {
*sboxOptions = append(
*sboxOptions,
libnetwork.OptionOriginHostsPath("/etc/hosts"),
)
originResolvConfPath = "/etc/resolv.conf"
case container.HostConfig.NetworkMode.IsUserDefined():
// The container uses a user-defined network. We use the embedded DNS
// server for container name resolution and to act as a DNS forwarder
// for external DNS resolution.
// We parse the DNS server(s) that are defined in /etc/resolv.conf on
// the host, which may be a local DNS server (for example, if DNSMasq or
// systemd-resolvd are in use). The embedded DNS server forwards DNS
// resolution to the DNS server configured on the host, which in itself
// may act as a forwarder for external DNS servers.
// If systemd-resolvd is used, the "upstream" DNS servers can be found in
// /run/systemd/resolve/resolv.conf. We do not query those DNS servers
// directly, as they can be dynamically reconfigured.
originResolvConfPath = "/etc/resolv.conf"
default:
// For other situations, such as the default bridge network, container
// discovery / name resolution is handled through /etc/hosts, and no
// embedded DNS server is available. Without the embedded DNS, we
// cannot use local DNS servers on the host (for example, if DNSMasq or
// systemd-resolvd is used). If systemd-resolvd is used, we try to
// determine the external DNS servers that are used on the host.
// This situation is not ideal, because DNS servers configured in the
// container are not updated after the container is created, but the
// DNS servers on the host can be dynamically updated.
//
// Copy the host's resolv.conf for the container (/run/systemd/resolve/resolv.conf or /etc/resolv.conf)
originResolvConfPath = cfg.GetResolvConf()
}
originResolvConfPath := "/etc/resolv.conf"
// Allow tests to point at their own resolv.conf file.
if envPath := os.Getenv("DOCKER_TEST_RESOLV_CONF_PATH"); envPath != "" {
log.G(context.TODO()).Infof("Using OriginResolvConfPath from env: %s", envPath)

View file

@ -809,9 +809,6 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
// Do we have a disabled network?
config.DisableBridge = isBridgeNetworkDisabled(config)
// Setup the resolv.conf
setupResolvConf(config)
idMapping, err := setupRemappedRoot(config)
if err != nil {
return nil, err

View file

@ -14,7 +14,6 @@ import (
"github.com/containerd/log"
"github.com/docker/docker/daemon/config"
"github.com/docker/docker/libnetwork/ns"
"github.com/docker/docker/libnetwork/resolvconf"
"github.com/moby/sys/mount"
"github.com/moby/sys/mountinfo"
"github.com/pkg/errors"
@ -136,18 +135,6 @@ func shouldUnmountRoot(root string, info *mountinfo.Info) bool {
return hasMountInfoOption(info.Optional, sharedPropagationOption)
}
// setupResolvConf sets the appropriate resolv.conf file if not specified
// When systemd-resolved is running the default /etc/resolv.conf points to
// localhost. In this case fetch the alternative config file that is in a
// different path so that containers can use it
// In all the other cases fallback to the default one
func setupResolvConf(config *config.Config) {
if config.ResolvConf != "" {
return
}
config.ResolvConf = resolvconf.Path()
}
// ifaceAddrs returns the IPv4 and IPv6 addresses assigned to the network
// interface with name linkName.
//

View file

@ -12,8 +12,6 @@ func checkSystem() error {
return errors.New("the Docker daemon is not supported on this platform")
}
func setupResolvConf(_ *interface{}) {}
func getSysInfo(_ *Daemon) *sysinfo.SysInfo {
return sysinfo.New()
}

View file

@ -555,8 +555,6 @@ func (daemon *Daemon) setupSeccompProfile(*config.Config) error {
return nil
}
func setupResolvConf(config *config.Config) {}
func getSysInfo(*config.Config) *sysinfo.SysInfo {
return sysinfo.New()
}

View file

@ -870,7 +870,8 @@ func buildCreateEndpointOptions(c *container.Container, n *libnetwork.Network, e
createOptions = append(createOptions, libnetwork.CreateOptionService(svcCfg.Name, svcCfg.ID, vip, portConfigs, svcCfg.Aliases[nwID]))
}
if !containertypes.NetworkMode(nwName).IsUserDefined() {
// Don't run an internal DNS resolver for host/container/none networks.
if nm := containertypes.NetworkMode(nwName); nm.IsHost() || nm.IsContainer() || nm.IsNone() {
createOptions = append(createOptions, libnetwork.CreateOptionDisableResolution())
}

View file

@ -27,7 +27,6 @@ import (
"github.com/docker/docker/integration-cli/cli/build"
"github.com/docker/docker/integration-cli/daemon"
"github.com/docker/docker/internal/testutils/specialimage"
"github.com/docker/docker/libnetwork/resolvconf"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/runconfig"
"github.com/docker/docker/testutil"
@ -1249,45 +1248,8 @@ func (s *DockerCLIRunSuite) TestRunDisallowBindMountingRootToRoot(c *testing.T)
}
}
// Verify that a container gets default DNS when only localhost resolvers exist
func (s *DockerCLIRunSuite) TestRunDNSDefaultOptions(c *testing.T) {
// Not applicable on Windows as this is testing Unix specific functionality
testRequires(c, testEnv.IsLocalDaemon, DaemonIsLinux)
// preserve original resolv.conf for restoring after test
origResolvConf, err := os.ReadFile("/etc/resolv.conf")
if os.IsNotExist(err) {
c.Fatalf("/etc/resolv.conf does not exist")
}
// defer restored original conf
defer func() {
if err := os.WriteFile("/etc/resolv.conf", origResolvConf, 0o644); err != nil {
c.Fatal(err)
}
}()
// test 3 cases: standard IPv4 localhost, commented out localhost, and IPv6 localhost
// 2 are removed from the file at container start, and the 3rd (commented out) one is ignored by
// GetNameservers(), leading to a replacement of nameservers with the default set
tmpResolvConf := []byte("nameserver 127.0.0.1\n#nameserver 127.0.2.1\nnameserver ::1")
if err := os.WriteFile("/etc/resolv.conf", tmpResolvConf, 0o644); err != nil {
c.Fatal(err)
}
actual := cli.DockerCmd(c, "run", "busybox", "cat", "/etc/resolv.conf").Combined()
actual = regexp.MustCompile("(?m)^#.*$").ReplaceAllString(actual, "")
actual = strings.ReplaceAll(strings.Trim(actual, "\r\n"), "\n", " ")
// NOTE: if we ever change the defaults from google dns, this will break
expected := "nameserver 8.8.8.8 nameserver 8.8.4.4"
if actual != expected {
c.Fatalf("expected resolv.conf be: %q, but was: %q", expected, actual)
}
}
func (s *DockerCLIRunSuite) TestRunDNSOptions(c *testing.T) {
// Not applicable on Windows as Windows does not support --dns*, or
// the Unix-specific functionality of resolv.conf.
// Not applicable on Windows as Windows does not support the Unix-specific functionality of resolv.conf.
testRequires(c, DaemonIsLinux)
result := cli.DockerCmd(c, "run", "--dns=127.0.0.1", "--dns-search=mydomain", "--dns-opt=ndots:9", "busybox", "cat", "/etc/resolv.conf")
@ -1298,16 +1260,22 @@ func (s *DockerCLIRunSuite) TestRunDNSOptions(c *testing.T) {
actual := regexp.MustCompile("(?m)^#.*$").ReplaceAllString(result.Stdout(), "")
actual = strings.ReplaceAll(strings.Trim(actual, "\r\n"), "\n", " ")
if actual != "nameserver 127.0.0.1 search mydomain options ndots:9" {
c.Fatalf("nameserver 127.0.0.1 expected 'search mydomain options ndots:9', but says: %q", actual)
if actual != "nameserver 127.0.0.11 search mydomain options ndots:9" {
c.Fatalf("nameserver 127.0.0.11 expected 'search mydomain options ndots:9', but says: %q", actual)
}
if !strings.Contains(result.Stdout(), "ExtServers: [127.0.0.1]") {
c.Fatalf("expected 'ExtServers: [127.0.0.1]' was not found in %q", result.Stdout())
}
out := cli.DockerCmd(c, "run", "--dns=1.1.1.1", "--dns-search=.", "--dns-opt=ndots:3", "busybox", "cat", "/etc/resolv.conf").Combined()
actual = regexp.MustCompile("(?m)^#.*$").ReplaceAllString(out, "")
actual = strings.ReplaceAll(strings.Trim(strings.Trim(actual, "\r\n"), " "), "\n", " ")
if actual != "nameserver 1.1.1.1 options ndots:3" {
c.Fatalf("expected 'nameserver 1.1.1.1 options ndots:3', but says: %q", actual)
if actual != "nameserver 127.0.0.11 options ndots:3" {
c.Fatalf("expected 'nameserver 127.0.0.11 options ndots:3', but says: %q", actual)
}
if !strings.Contains(out, "ExtServers: [1.1.1.1]") {
c.Fatalf("expected 'ExtServers: [1.1.1.1]' was not found in %q", out)
}
}
@ -1317,87 +1285,11 @@ func (s *DockerCLIRunSuite) TestRunDNSRepeatOptions(c *testing.T) {
actual := regexp.MustCompile("(?m)^#.*$").ReplaceAllString(out, "")
actual = strings.ReplaceAll(strings.Trim(actual, "\r\n"), "\n", " ")
if actual != "nameserver 1.1.1.1 nameserver 2.2.2.2 search mydomain mydomain2 options ndots:9 timeout:3" {
c.Fatalf("expected 'nameserver 1.1.1.1 nameserver 2.2.2.2 search mydomain mydomain2 options ndots:9 timeout:3', but says: %q", actual)
if actual != "nameserver 127.0.0.11 search mydomain mydomain2 options ndots:9 timeout:3" {
c.Fatalf("expected 'nameserver 127.0.0.11 search mydomain mydomain2 options ndots:9 timeout:3', but says: %q", actual)
}
}
func (s *DockerCLIRunSuite) TestRunDNSOptionsBasedOnHostResolvConf(c *testing.T) {
// Not applicable on Windows as testing Unix specific functionality
testRequires(c, testEnv.IsLocalDaemon, DaemonIsLinux)
origResolvConf, err := os.ReadFile("/etc/resolv.conf")
if os.IsNotExist(err) {
c.Fatalf("/etc/resolv.conf does not exist")
}
hostNameservers := resolvconf.GetNameservers(origResolvConf, resolvconf.IP)
hostSearch := resolvconf.GetSearchDomains(origResolvConf)
out := cli.DockerCmd(c, "run", "--dns=127.0.0.1", "busybox", "cat", "/etc/resolv.conf").Combined()
if actualNameservers := resolvconf.GetNameservers([]byte(out), resolvconf.IP); actualNameservers[0] != "127.0.0.1" {
c.Fatalf("expected '127.0.0.1', but says: %q", actualNameservers[0])
}
actualSearch := resolvconf.GetSearchDomains([]byte(out))
if len(actualSearch) != len(hostSearch) {
c.Fatalf("expected %q search domain(s), but it has: %q", len(hostSearch), len(actualSearch))
}
for i := range actualSearch {
if actualSearch[i] != hostSearch[i] {
c.Fatalf("expected %q domain, but says: %q", actualSearch[i], hostSearch[i])
}
}
out = cli.DockerCmd(c, "run", "--dns-search=mydomain", "busybox", "cat", "/etc/resolv.conf").Combined()
actualNameservers := resolvconf.GetNameservers([]byte(out), resolvconf.IP)
if len(actualNameservers) != len(hostNameservers) {
c.Fatalf("expected %q nameserver(s), but it has: %q", len(hostNameservers), len(actualNameservers))
}
for i := range actualNameservers {
if actualNameservers[i] != hostNameservers[i] {
c.Fatalf("expected %q nameserver, but says: %q", actualNameservers[i], hostNameservers[i])
}
}
if actualSearch = resolvconf.GetSearchDomains([]byte(out)); actualSearch[0] != "mydomain" {
c.Fatalf("expected 'mydomain', but says: %q", actualSearch[0])
}
// test with file
tmpResolvConf := []byte("search example.com\nnameserver 12.34.56.78\nnameserver 127.0.0.1")
if err := os.WriteFile("/etc/resolv.conf", tmpResolvConf, 0o644); err != nil {
c.Fatal(err)
}
// put the old resolvconf back
defer func() {
if err := os.WriteFile("/etc/resolv.conf", origResolvConf, 0o644); err != nil {
c.Fatal(err)
}
}()
resolvConf, err := os.ReadFile("/etc/resolv.conf")
if os.IsNotExist(err) {
c.Fatalf("/etc/resolv.conf does not exist")
}
hostSearch = resolvconf.GetSearchDomains(resolvConf)
out = cli.DockerCmd(c, "run", "busybox", "cat", "/etc/resolv.conf").Combined()
if actualNameservers = resolvconf.GetNameservers([]byte(out), resolvconf.IP); actualNameservers[0] != "12.34.56.78" || len(actualNameservers) != 1 {
c.Fatalf("expected '12.34.56.78', but has: %v", actualNameservers)
}
actualSearch = resolvconf.GetSearchDomains([]byte(out))
if len(actualSearch) != len(hostSearch) {
c.Fatalf("expected %q search domain(s), but it has: %q", len(hostSearch), len(actualSearch))
}
for i := range actualSearch {
if actualSearch[i] != hostSearch[i] {
c.Fatalf("expected %q domain, but says: %q", actualSearch[i], hostSearch[i])
}
if !strings.Contains(out, "ExtServers: [1.1.1.1 2.2.2.2]") {
c.Fatalf("expected 'ExtServers: [127.0.0.1]' was not found in %q", out)
}
}

View file

@ -972,14 +972,16 @@ func (s *DockerSwarmSuite) TestDNSConfig(c *testing.T) {
id := strings.TrimSpace(out)
// Compare against expected output.
expectedOutput1 := "nameserver 1.2.3.4"
expectedOutput1 := "nameserver 127.0.0.11"
expectedOutput2 := "search example.com"
expectedOutput3 := "options timeout:3"
expectedOutput4 := "ExtServers: [1.2.3.4]"
out, err = d.Cmd("exec", id, "cat", "/etc/resolv.conf")
assert.NilError(c, err, out)
assert.Assert(c, strings.Contains(out, expectedOutput1), "Expected '%s', but got %q", expectedOutput1, out)
assert.Assert(c, strings.Contains(out, expectedOutput2), "Expected '%s', but got %q", expectedOutput2, out)
assert.Assert(c, strings.Contains(out, expectedOutput3), "Expected '%s', but got %q", expectedOutput3, out)
assert.Assert(c, strings.Contains(out, expectedOutput4), "Expected '%s', but got %q", expectedOutput4, out)
}
func (s *DockerSwarmSuite) TestDNSConfigUpdate(c *testing.T) {

View file

@ -7,6 +7,7 @@ import (
"testing"
containertypes "github.com/docker/docker/api/types/container"
networktypes "github.com/docker/docker/api/types/network"
"github.com/docker/docker/integration/internal/container"
"github.com/docker/docker/integration/internal/network"
"github.com/docker/docker/testutil/daemon"
@ -189,3 +190,52 @@ func TestInternalNetworkDNS(t *testing.T) {
assert.Check(t, is.Equal(res.ExitCode, 0))
assert.Check(t, is.Contains(res.Stdout(), dnsRespAddr))
}
// Check that containers on the default bridge network can use a host's resolver
// running on a loopback interface (via the internal resolver), but the internal
// resolver is not populated with DNS names for containers (no service discovery
// on the legacy/default bridge network).
// (Assumes the host does not already have a DNS server on 127.0.0.1.)
func TestDefaultBridgeDNS(t *testing.T) {
skip.If(t, testEnv.DaemonInfo.OSType == "windows", "No resolv.conf on Windows")
skip.If(t, testEnv.IsRootless, "Can't use resolver on host in rootless mode")
ctx := setupTest(t)
// Start a DNS server on the loopback interface.
server := startDaftDNS(t, "127.0.0.1")
defer server.Shutdown()
// Set up a temp resolv.conf pointing at that DNS server, and a daemon using it.
tmpFileName := writeTempResolvConf(t, "127.0.0.1")
d := daemon.New(t, daemon.WithEnvVars("DOCKER_TEST_RESOLV_CONF_PATH="+tmpFileName))
d.StartWithBusybox(ctx, t)
defer d.Stop(t)
c := d.NewClientT(t)
defer c.Close()
// Create a container on the default bridge network.
const ctrName = "ctrname"
ctrId := container.Run(ctx, t, c, container.WithName(ctrName))
defer c.ContainerRemove(ctx, ctrId, containertypes.RemoveOptions{Force: true})
// Expect the external DNS server to respond to a request from the container.
res, err := container.Exec(ctx, c, ctrId, []string{"nslookup", "test.example"})
assert.NilError(t, err)
assert.Check(t, is.Equal(res.ExitCode, 0))
assert.Check(t, is.Contains(res.Stdout(), dnsRespAddr))
// Expect the external DNS server to respond to a request from the container
// for the container's own name - it won't be recognised as a container name
// because there's no service resolution on the default bridge.
res, err = container.Exec(ctx, c, ctrId, []string{"nslookup", ctrName})
assert.NilError(t, err)
assert.Check(t, is.Equal(res.ExitCode, 0))
assert.Check(t, is.Contains(res.Stdout(), dnsRespAddr))
// Check that inspect output has no DNSNames for the container.
inspect := container.Inspect(ctx, t, c, ctrId)
net, ok := inspect.NetworkSettings.Networks[networktypes.NetworkBridge]
assert.Check(t, ok, "expected to find bridge network in inspect output")
assert.Check(t, is.Nil(net.DNSNames))
}

View file

@ -525,9 +525,6 @@ func (ep *Endpoint) sbJoin(sb *Sandbox, options ...EndpointOption) (err error) {
if err := sb.updateHostsFile(ep.getEtcHostsAddrs()); err != nil {
return err
}
if err = sb.updateDNS(n.enableIPv6); err != nil {
return err
}
// Current endpoint providing external connectivity for the sandbox
extEp := sb.getGatewayEndpoint()

View file

@ -294,27 +294,6 @@ func (sb *Sandbox) setupDNS() error {
return rc.WriteFile(sb.config.resolvConfPath, sb.config.resolvConfHashFile, filePerm)
}
// Called when an endpoint has joined the sandbox.
func (sb *Sandbox) updateDNS(ipv6Enabled bool) error {
if mod, err := resolvconf.UserModified(sb.config.resolvConfPath, sb.config.resolvConfHashFile); err != nil || mod {
return err
}
// Load the host's resolv.conf as a starting point.
rc, err := sb.loadResolvConf(sb.config.getOriginResolvConfPath())
if err != nil {
return err
}
// For host-networking, no further change is needed.
if !sb.config.useDefaultSandBox {
// The legacy bridge network has no internal nameserver. So, strip localhost
// nameservers from the host's config, then add default nameservers if there
// are none remaining.
rc.TransformForLegacyNw(ipv6Enabled)
}
return rc.WriteFile(sb.config.resolvConfPath, sb.config.resolvConfHashFile, filePerm)
}
// Embedded DNS server has to be enabled for this sandbox. Rebuild the container's resolv.conf.
func (sb *Sandbox) rebuildDNS() error {
// Don't touch the file if the user has modified it.