diff --git a/api/server/server.go b/api/server/server.go index a7d0a58b2f..96abf5c3b5 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -27,7 +27,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/api" "github.com/docker/docker/api/types" - "github.com/docker/docker/daemon/networkdriver/portallocator" + "github.com/docker/docker/daemon/networkdriver/bridge" "github.com/docker/docker/engine" "github.com/docker/docker/pkg/listenbuffer" "github.com/docker/docker/pkg/parsers" @@ -1542,7 +1542,7 @@ func allocateDaemonPort(addr string) error { } for _, hostIP := range hostIPs { - if _, err := portallocator.RequestPort(hostIP, "tcp", intPort); err != nil { + if _, err := bridge.RequestPort(hostIP, "tcp", intPort); err != nil { return fmt.Errorf("failed to allocate daemon listening port %d (err: %v)", intPort, err) } } diff --git a/daemon/daemon.go b/daemon/daemon.go index ba9c73ce64..36cf438bbb 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -25,7 +25,6 @@ import ( "github.com/docker/docker/daemon/graphdriver" _ "github.com/docker/docker/daemon/graphdriver/vfs" _ "github.com/docker/docker/daemon/networkdriver/bridge" - "github.com/docker/docker/daemon/networkdriver/portallocator" "github.com/docker/docker/engine" "github.com/docker/docker/graph" "github.com/docker/docker/image" @@ -818,12 +817,6 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) } config.DisableNetwork = config.BridgeIface == disableNetworkBridge - // register portallocator release on shutdown - eng.OnShutdown(func() { - if err := portallocator.ReleaseAll(); err != nil { - logrus.Errorf("portallocator.ReleaseAll(): %s", err) - } - }) // Claim the pidfile first, to avoid any and all unexpected race conditions. // Some of the init doesn't need a pidfile lock - but let's not try to be smart. if config.Pidfile != "" { diff --git a/daemon/networkdriver/bridge/driver.go b/daemon/networkdriver/bridge/driver.go index c5f5704f94..21d8c44554 100644 --- a/daemon/networkdriver/bridge/driver.go +++ b/daemon/networkdriver/bridge/driver.go @@ -77,6 +77,7 @@ var ( bridgeIPv4Network *net.IPNet bridgeIPv6Addr net.IP globalIPv6Network *net.IPNet + portMapper *portmapper.PortMapper defaultBindingIP = net.ParseIP("0.0.0.0") currentInterfaces = ifaces{c: make(map[string]*networkInterface)} @@ -99,6 +100,7 @@ func InitDriver(job *engine.Job) error { fixedCIDR = job.Getenv("FixedCIDR") fixedCIDRv6 = job.Getenv("FixedCIDRv6") ) + portMapper = portmapper.New() if defaultIP := job.Getenv("DefaultBindingIP"); defaultIP != "" { defaultBindingIP = net.ParseIP(defaultIP) @@ -235,7 +237,7 @@ func InitDriver(job *engine.Job) error { if err != nil { return err } - portmapper.SetIptablesChain(chain) + portMapper.SetIptablesChain(chain) } bridgeIPv4Network = networkv4 @@ -350,6 +352,10 @@ func setupIPTables(addr net.Addr, icc, ipmasq bool) error { return nil } +func RequestPort(ip net.IP, proto string, port int) (int, error) { + return portMapper.Allocator.RequestPort(ip, proto, port) +} + // configureBridge attempts to create and configure a network bridge interface named `bridgeIface` on the host // If bridgeIP is empty, it will try to find a non-conflicting IP from the Docker-specified private ranges // If the bridge `bridgeIface` already exists, it will only perform the IP address association with the existing @@ -587,7 +593,7 @@ func Release(job *engine.Job) error { } for _, nat := range containerInterface.PortMappings { - if err := portmapper.Unmap(nat); err != nil { + if err := portMapper.Unmap(nat); err != nil { logrus.Infof("Unable to unmap port %s: %s", nat, err) } } @@ -644,7 +650,7 @@ func AllocatePort(job *engine.Job) error { var host net.Addr for i := 0; i < MaxAllocatedPortAttempts; i++ { - if host, err = portmapper.Map(container, ip, hostPort); err == nil { + if host, err = portMapper.Map(container, ip, hostPort); err == nil { break } // There is no point in immediately retrying to map an explicitly diff --git a/daemon/networkdriver/portallocator/portallocator.go b/daemon/networkdriver/portallocator/portallocator.go index e2bb9ee56b..c1f414b673 100644 --- a/daemon/networkdriver/portallocator/portallocator.go +++ b/daemon/networkdriver/portallocator/portallocator.go @@ -16,59 +16,14 @@ const ( DefaultPortRangeEnd = 65535 ) -var ( - beginPortRange = DefaultPortRangeStart - endPortRange = DefaultPortRangeEnd -) - -type portMap struct { - p map[int]struct{} - last int -} - -func newPortMap() *portMap { - return &portMap{ - p: map[int]struct{}{}, - last: endPortRange, - } -} - -type protoMap map[string]*portMap - -func newProtoMap() protoMap { - return protoMap{ - "tcp": newPortMap(), - "udp": newPortMap(), - } -} - type ipMapping map[string]protoMap var ( ErrAllPortsAllocated = errors.New("all ports are allocated") ErrUnknownProtocol = errors.New("unknown protocol") + defaultIP = net.ParseIP("0.0.0.0") ) -var ( - defaultIP = net.ParseIP("0.0.0.0") - - DefaultPortAllocator = New() - RequestPort = DefaultPortAllocator.RequestPort - ReleasePort = DefaultPortAllocator.ReleasePort - ReleaseAll = DefaultPortAllocator.ReleaseAll -) - -type PortAllocator struct { - mutex sync.Mutex - ipMap ipMapping -} - -func New() *PortAllocator { - return &PortAllocator{ - ipMap: ipMapping{}, - } -} - type ErrPortAlreadyAllocated struct { ip string port int @@ -81,32 +36,6 @@ func NewErrPortAlreadyAllocated(ip string, port int) ErrPortAlreadyAllocated { } } -func init() { - const portRangeKernelParam = "/proc/sys/net/ipv4/ip_local_port_range" - portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", beginPortRange, endPortRange) - - file, err := os.Open(portRangeKernelParam) - if err != nil { - logrus.Warnf("port allocator - %s due to error: %v", portRangeFallback, err) - return - } - var start, end int - n, err := fmt.Fscanf(bufio.NewReader(file), "%d\t%d", &start, &end) - if n != 2 || err != nil { - if err == nil { - err = fmt.Errorf("unexpected count of parsed numbers (%d)", n) - } - logrus.Errorf("port allocator - failed to parse system ephemeral port range from %s - %s: %v", portRangeKernelParam, portRangeFallback, err) - return - } - beginPortRange = start - endPortRange = end -} - -func PortRange() (int, int) { - return beginPortRange, endPortRange -} - func (e ErrPortAlreadyAllocated) IP() string { return e.ip } @@ -123,6 +52,51 @@ func (e ErrPortAlreadyAllocated) Error() string { return fmt.Sprintf("Bind for %s:%d failed: port is already allocated", e.ip, e.port) } +type ( + PortAllocator struct { + mutex sync.Mutex + ipMap ipMapping + Begin int + End int + } + portMap struct { + p map[int]struct{} + begin, end int + last int + } + protoMap map[string]*portMap +) + +func New() *PortAllocator { + start, end, err := getDynamicPortRange() + if err != nil { + logrus.Warn(err) + start, end = DefaultPortRangeStart, DefaultPortRangeEnd + } + return &PortAllocator{ + ipMap: ipMapping{}, + Begin: start, + End: end, + } +} + +func getDynamicPortRange() (start int, end int, err error) { + const portRangeKernelParam = "/proc/sys/net/ipv4/ip_local_port_range" + portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", DefaultPortRangeStart, DefaultPortRangeEnd) + file, err := os.Open(portRangeKernelParam) + if err != nil { + return 0, 0, fmt.Errorf("port allocator - %s due to error: %v", portRangeFallback, err) + } + n, err := fmt.Fscanf(bufio.NewReader(file), "%d\t%d", &start, &end) + if n != 2 || err != nil { + if err == nil { + err = fmt.Errorf("unexpected count of parsed numbers (%d)", n) + } + return 0, 0, fmt.Errorf("port allocator - failed to parse system ephemeral port range from %s - %s: %v", portRangeKernelParam, portRangeFallback, err) + } + return start, end, nil +} + // RequestPort requests new port from global ports pool for specified ip and proto. // If port is 0 it returns first free port. Otherwise it cheks port availability // in pool and return that port or error if port is already busy. @@ -140,7 +114,11 @@ func (p *PortAllocator) RequestPort(ip net.IP, proto string, port int) (int, err ipstr := ip.String() protomap, ok := p.ipMap[ipstr] if !ok { - protomap = newProtoMap() + protomap = protoMap{ + "tcp": p.newPortMap(), + "udp": p.newPortMap(), + } + p.ipMap[ipstr] = protomap } mapping := protomap[proto] @@ -175,6 +153,15 @@ func (p *PortAllocator) ReleasePort(ip net.IP, proto string, port int) error { return nil } +func (p *PortAllocator) newPortMap() *portMap { + return &portMap{ + p: map[int]struct{}{}, + begin: p.Begin, + end: p.End, + last: p.End, + } +} + // ReleaseAll releases all ports for all ips. func (p *PortAllocator) ReleaseAll() error { p.mutex.Lock() @@ -185,10 +172,10 @@ func (p *PortAllocator) ReleaseAll() error { func (pm *portMap) findPort() (int, error) { port := pm.last - for i := 0; i <= endPortRange-beginPortRange; i++ { + for i := 0; i <= pm.end-pm.begin; i++ { port++ - if port > endPortRange { - port = beginPortRange + if port > pm.end { + port = pm.begin } if _, ok := pm.p[port]; !ok { diff --git a/daemon/networkdriver/portallocator/portallocator_test.go b/daemon/networkdriver/portallocator/portallocator_test.go index f6f122bbde..17201235e0 100644 --- a/daemon/networkdriver/portallocator/portallocator_test.go +++ b/daemon/networkdriver/portallocator/portallocator_test.go @@ -5,11 +5,6 @@ import ( "testing" ) -func init() { - beginPortRange = DefaultPortRangeStart - endPortRange = DefaultPortRangeEnd -} - func TestRequestNewPort(t *testing.T) { p := New() @@ -18,7 +13,7 @@ func TestRequestNewPort(t *testing.T) { t.Fatal(err) } - if expected := beginPortRange; port != expected { + if expected := p.Begin; port != expected { t.Fatalf("Expected port %d got %d", expected, port) } } @@ -101,13 +96,13 @@ func TestUnknowProtocol(t *testing.T) { func TestAllocateAllPorts(t *testing.T) { p := New() - for i := 0; i <= endPortRange-beginPortRange; i++ { + for i := 0; i <= p.End-p.Begin; i++ { port, err := p.RequestPort(defaultIP, "tcp", 0) if err != nil { t.Fatal(err) } - if expected := beginPortRange + i; port != expected { + if expected := p.Begin + i; port != expected { t.Fatalf("Expected port %d got %d", expected, port) } } @@ -122,7 +117,7 @@ func TestAllocateAllPorts(t *testing.T) { } // release a port in the middle and ensure we get another tcp port - port := beginPortRange + 5 + port := p.Begin + 5 if err := p.ReleasePort(defaultIP, "tcp", port); err != nil { t.Fatal(err) } @@ -152,13 +147,13 @@ func BenchmarkAllocatePorts(b *testing.B) { p := New() for i := 0; i < b.N; i++ { - for i := 0; i <= endPortRange-beginPortRange; i++ { + for i := 0; i <= p.End-p.Begin; i++ { port, err := p.RequestPort(defaultIP, "tcp", 0) if err != nil { b.Fatal(err) } - if expected := beginPortRange + i; port != expected { + if expected := p.Begin + i; port != expected { b.Fatalf("Expected port %d got %d", expected, port) } } @@ -230,15 +225,15 @@ func TestPortAllocation(t *testing.T) { func TestNoDuplicateBPR(t *testing.T) { p := New() - if port, err := p.RequestPort(defaultIP, "tcp", beginPortRange); err != nil { + if port, err := p.RequestPort(defaultIP, "tcp", p.Begin); err != nil { t.Fatal(err) - } else if port != beginPortRange { - t.Fatalf("Expected port %d got %d", beginPortRange, port) + } else if port != p.Begin { + t.Fatalf("Expected port %d got %d", p.Begin, port) } if port, err := p.RequestPort(defaultIP, "tcp", 0); err != nil { t.Fatal(err) - } else if port == beginPortRange { + } else if port == p.Begin { t.Fatalf("Acquire(0) allocated the same port twice: %d", port) } } diff --git a/daemon/networkdriver/portmapper/mapper.go b/daemon/networkdriver/portmapper/mapper.go index a01b604160..8f79bae3f2 100644 --- a/daemon/networkdriver/portmapper/mapper.go +++ b/daemon/networkdriver/portmapper/mapper.go @@ -18,14 +18,7 @@ type mapping struct { container net.Addr } -var ( - NewProxy = NewProxyCommand - - DefaultPortMapper = NewWithPortAllocator(portallocator.DefaultPortAllocator) - SetIptablesChain = DefaultPortMapper.SetIptablesChain - Map = DefaultPortMapper.Map - Unmap = DefaultPortMapper.Unmap -) +var NewProxy = NewProxyCommand var ( ErrUnknownBackendAddressType = errors.New("unknown container address type not supported") @@ -40,7 +33,7 @@ type PortMapper struct { currentMappings map[string]*mapping lock sync.Mutex - allocator *portallocator.PortAllocator + Allocator *portallocator.PortAllocator } func New() *PortMapper { @@ -50,7 +43,7 @@ func New() *PortMapper { func NewWithPortAllocator(allocator *portallocator.PortAllocator) *PortMapper { return &PortMapper{ currentMappings: make(map[string]*mapping), - allocator: allocator, + Allocator: allocator, } } @@ -72,7 +65,7 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host switch container.(type) { case *net.TCPAddr: proto = "tcp" - if allocatedHostPort, err = pm.allocator.RequestPort(hostIP, proto, hostPort); err != nil { + if allocatedHostPort, err = pm.Allocator.RequestPort(hostIP, proto, hostPort); err != nil { return nil, err } @@ -85,7 +78,7 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host proxy = NewProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port) case *net.UDPAddr: proto = "udp" - if allocatedHostPort, err = pm.allocator.RequestPort(hostIP, proto, hostPort); err != nil { + if allocatedHostPort, err = pm.Allocator.RequestPort(hostIP, proto, hostPort); err != nil { return nil, err } @@ -103,7 +96,7 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host // release the allocated port on any further error during return. defer func() { if err != nil { - pm.allocator.ReleasePort(hostIP, proto, allocatedHostPort) + pm.Allocator.ReleasePort(hostIP, proto, allocatedHostPort) } }() @@ -121,7 +114,7 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host // need to undo the iptables rules before we return proxy.Stop() pm.forward(iptables.Delete, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort) - if err := pm.allocator.ReleasePort(hostIP, m.proto, allocatedHostPort); err != nil { + if err := pm.Allocator.ReleasePort(hostIP, m.proto, allocatedHostPort); err != nil { return err } @@ -161,9 +154,9 @@ func (pm *PortMapper) Unmap(host net.Addr) error { switch a := host.(type) { case *net.TCPAddr: - return pm.allocator.ReleasePort(a.IP, "tcp", a.Port) + return pm.Allocator.ReleasePort(a.IP, "tcp", a.Port) case *net.UDPAddr: - return pm.allocator.ReleasePort(a.IP, "udp", a.Port) + return pm.Allocator.ReleasePort(a.IP, "udp", a.Port) } return nil } diff --git a/daemon/networkdriver/portmapper/mapper_test.go b/daemon/networkdriver/portmapper/mapper_test.go index 4082a6002b..729fe56075 100644 --- a/daemon/networkdriver/portmapper/mapper_test.go +++ b/daemon/networkdriver/portmapper/mapper_test.go @@ -4,7 +4,6 @@ import ( "net" "testing" - "github.com/docker/docker/daemon/networkdriver/portallocator" "github.com/docker/docker/pkg/iptables" ) @@ -126,7 +125,7 @@ func TestMapAllPortsSingleInterface(t *testing.T) { }() for i := 0; i < 10; i++ { - start, end := portallocator.PortRange() + start, end := pm.Allocator.Begin, pm.Allocator.End for i := start; i < end; i++ { if host, err = pm.Map(srcAddr1, dstIp1, 0); err != nil { t.Fatal(err)