diff --git a/libnetwork/drivers/windows/windows.go b/libnetwork/drivers/windows/windows.go index 84089a5284..c1c69f2991 100644 --- a/libnetwork/drivers/windows/windows.go +++ b/libnetwork/drivers/windows/windows.go @@ -41,6 +41,8 @@ type networkConfiguration struct { DNSSuffix string SourceMac string NetworkAdapterName string + dbIndex uint64 + dbExists bool } // endpointConfiguration represents the user specified configuration for the sandbox endpoint @@ -59,17 +61,22 @@ type endpointConnectivity struct { type hnsEndpoint struct { id string + nid string profileID string + Type string macAddress net.HardwareAddr epOption *endpointOption // User specified parameters epConnectivity *endpointConnectivity // User specified parameters portMapping []types.PortBinding // Operation port bindings addr *net.IPNet gateway net.IP + dbIndex uint64 + dbExists bool } type hnsNetwork struct { id string + created bool config *networkConfiguration endpoints map[string]*hnsEndpoint // key: endpoint id driver *driver // The network's driver @@ -79,9 +86,14 @@ type hnsNetwork struct { type driver struct { name string networks map[string]*hnsNetwork + store datastore.DataStore sync.Mutex } +const ( + errNotFound = "HNS failed with error : The object identifier does not represent a valid object. " +) + // IsBuiltinWindowsDriver vaidates if network-type is a builtin local-scoped driver func IsBuiltinLocalDriver(networkType string) bool { if "l2bridge" == networkType || "l2tunnel" == networkType || "nat" == networkType || "ics" == networkType || "transparent" == networkType { @@ -103,7 +115,14 @@ func GetInit(networkType string) func(dc driverapi.DriverCallback, config map[st return types.BadRequestErrorf("Network type not supported: %s", networkType) } - return dc.RegisterDriver(networkType, newDriver(networkType), driverapi.Capability{ + d := newDriver(networkType) + + err := d.initStore(config) + if err != nil { + return err + } + + return dc.RegisterDriver(networkType, d, driverapi.Capability{ DataScope: datastore.LocalScope, ConnectivityScope: datastore.LocalScope, }) @@ -133,7 +152,7 @@ func (n *hnsNetwork) getEndpoint(eid string) (*hnsEndpoint, error) { } func (d *driver) parseNetworkOptions(id string, genericOptions map[string]string) (*networkConfiguration, error) { - config := &networkConfiguration{} + config := &networkConfiguration{Type: d.name} for label, value := range genericOptions { switch label { @@ -188,6 +207,21 @@ func (d *driver) DecodeTableEntry(tablename string, key string, value []byte) (s return "", nil } +func (d *driver) createNetwork(config *networkConfiguration) error { + network := &hnsNetwork{ + id: config.ID, + endpoints: make(map[string]*hnsEndpoint), + config: config, + driver: d, + } + + d.Lock() + d.networks[config.ID] = network + d.Unlock() + + return nil +} + // Create a new network func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error { if _, err := d.getNetwork(id); err == nil { @@ -210,16 +244,11 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d return err } - network := &hnsNetwork{ - id: config.ID, - endpoints: make(map[string]*hnsEndpoint), - config: config, - driver: d, - } + err = d.createNetwork(config) - d.Lock() - d.networks[config.ID] = network - d.Unlock() + if err != nil { + return err + } // A non blank hnsid indicates that the network was discovered // from HNS. No need to call HNS if this network was discovered @@ -294,7 +323,9 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d genData[HNSID] = config.HnsID } - return nil + n, err := d.getNetwork(id) + n.created = true + return d.storeUpdate(config) } func (d *driver) DeleteNetwork(nid string) error { @@ -307,21 +338,25 @@ func (d *driver) DeleteNetwork(nid string) error { config := n.config n.Unlock() - // Cannot remove network if endpoints are still present - if len(n.endpoints) != 0 { - return fmt.Errorf("network %s has active endpoint", n.id) - } - - _, err = hcsshim.HNSNetworkRequest("DELETE", config.HnsID, "") - if err != nil { - return types.ForbiddenErrorf(err.Error()) + if n.created { + _, err = hcsshim.HNSNetworkRequest("DELETE", config.HnsID, "") + if err != nil && err.Error() != errNotFound { + return types.ForbiddenErrorf(err.Error()) + } } d.Lock() delete(d.networks, nid) d.Unlock() - return nil + // delele endpoints belong to this network + for _, ep := range n.endpoints { + if err := d.storeDelete(ep); err != nil { + logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err) + } + } + + return d.storeDelete(config) } func convertQosPolicies(qosPolicies []types.QosPolicy) ([]json.RawMessage, error) { @@ -544,6 +579,8 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, // TODO For now the ip mask is not in the info generated by HNS endpoint := &hnsEndpoint{ id: eid, + nid: n.id, + Type: d.name, addr: &net.IPNet{IP: hnsresponse.IPAddress, Mask: hnsresponse.IPAddress.DefaultMask()}, macAddress: mac, } @@ -574,6 +611,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, ifInfo.SetMacAddress(endpoint.macAddress) } + if err = d.storeUpdate(endpoint); err != nil { + return fmt.Errorf("failed to save endpoint %s to store: %v", endpoint.id[0:7], err) + } + return nil } @@ -593,10 +634,13 @@ func (d *driver) DeleteEndpoint(nid, eid string) error { n.Unlock() _, err = hcsshim.HNSEndpointRequest("DELETE", ep.profileID, "") - if err != nil { + if err != nil && err.Error() != errNotFound { return err } + if err := d.storeDelete(ep); err != nil { + logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err) + } return nil } diff --git a/libnetwork/drivers/windows/windows_store.go b/libnetwork/drivers/windows/windows_store.go new file mode 100644 index 0000000000..3c5cb04e21 --- /dev/null +++ b/libnetwork/drivers/windows/windows_store.go @@ -0,0 +1,335 @@ +// +build windows + +package windows + +import ( + "encoding/json" + "fmt" + "net" + + "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/discoverapi" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/types" +) + +const ( + windowsPrefix = "windows" + windowsEndpointPrefix = "windows-endpoint" +) + +func (d *driver) initStore(option map[string]interface{}) error { + if data, ok := option[netlabel.LocalKVClient]; ok { + var err error + dsc, ok := data.(discoverapi.DatastoreConfigData) + if !ok { + return types.InternalErrorf("incorrect data in datastore configuration: %v", data) + } + d.store, err = datastore.NewDataStoreFromConfig(dsc) + if err != nil { + return types.InternalErrorf("windows driver failed to initialize data store: %v", err) + } + + err = d.populateNetworks() + if err != nil { + return err + } + + err = d.populateEndpoints() + if err != nil { + return err + } + } + + return nil +} + +func (d *driver) populateNetworks() error { + kvol, err := d.store.List(datastore.Key(windowsPrefix), &networkConfiguration{Type: d.name}) + if err != nil && err != datastore.ErrKeyNotFound { + return fmt.Errorf("failed to get windows network configurations from store: %v", err) + } + + // It's normal for network configuration state to be empty. Just return. + if err == datastore.ErrKeyNotFound { + return nil + } + + for _, kvo := range kvol { + ncfg := kvo.(*networkConfiguration) + if ncfg.Type != d.name { + continue + } + if err = d.createNetwork(ncfg); err != nil { + logrus.Warnf("could not create windows network for id %s hnsid %s while booting up from persistent state: %v", ncfg.ID, ncfg.HnsID, err) + } + logrus.Debugf("Network (%s) restored", ncfg.ID[0:7]) + } + + return nil +} + +func (d *driver) populateEndpoints() error { + kvol, err := d.store.List(datastore.Key(windowsEndpointPrefix), &hnsEndpoint{Type: d.name}) + if err != nil && err != datastore.ErrKeyNotFound { + return fmt.Errorf("failed to get endpoints from store: %v", err) + } + + if err == datastore.ErrKeyNotFound { + return nil + } + + for _, kvo := range kvol { + ep := kvo.(*hnsEndpoint) + if ep.Type != d.name { + continue + } + n, ok := d.networks[ep.nid] + if !ok { + logrus.Debugf("Network (%s) not found for restored endpoint (%s)", ep.nid[0:7], ep.id[0:7]) + logrus.Debugf("Deleting stale endpoint (%s) from store", ep.id[0:7]) + if err := d.storeDelete(ep); err != nil { + logrus.Debugf("Failed to delete stale endpoint (%s) from store", ep.id[0:7]) + } + continue + } + n.endpoints[ep.id] = ep + logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7]) + } + + return nil +} + +func (d *driver) storeUpdate(kvObject datastore.KVObject) error { + if d.store == nil { + logrus.Warnf("store not initialized. kv object %s is not added to the store", datastore.Key(kvObject.Key()...)) + return nil + } + + if err := d.store.PutObjectAtomic(kvObject); err != nil { + return fmt.Errorf("failed to update store for object type %T: %v", kvObject, err) + } + + return nil +} + +func (d *driver) storeDelete(kvObject datastore.KVObject) error { + if d.store == nil { + logrus.Debugf("store not initialized. kv object %s is not deleted from store", datastore.Key(kvObject.Key()...)) + return nil + } + +retry: + if err := d.store.DeleteObjectAtomic(kvObject); err != nil { + if err == datastore.ErrKeyModified { + if err := d.store.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil { + return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err) + } + goto retry + } + return err + } + + return nil +} + +func (ncfg *networkConfiguration) MarshalJSON() ([]byte, error) { + nMap := make(map[string]interface{}) + + nMap["ID"] = ncfg.ID + nMap["Type"] = ncfg.Type + nMap["Name"] = ncfg.Name + nMap["HnsID"] = ncfg.HnsID + nMap["VLAN"] = ncfg.VLAN + nMap["VSID"] = ncfg.VSID + nMap["DNSServers"] = ncfg.DNSServers + nMap["DNSSuffix"] = ncfg.DNSSuffix + nMap["SourceMac"] = ncfg.SourceMac + nMap["NetworkAdapterName"] = ncfg.NetworkAdapterName + + return json.Marshal(nMap) +} + +func (ncfg *networkConfiguration) UnmarshalJSON(b []byte) error { + var ( + err error + nMap map[string]interface{} + ) + + if err = json.Unmarshal(b, &nMap); err != nil { + return err + } + + ncfg.ID = nMap["ID"].(string) + ncfg.Type = nMap["Type"].(string) + ncfg.Name = nMap["Name"].(string) + ncfg.HnsID = nMap["HnsID"].(string) + ncfg.VLAN = uint(nMap["VLAN"].(float64)) + ncfg.VSID = uint(nMap["VSID"].(float64)) + ncfg.DNSServers = nMap["DNSServers"].(string) + ncfg.DNSSuffix = nMap["DNSSuffix"].(string) + ncfg.SourceMac = nMap["SourceMac"].(string) + ncfg.NetworkAdapterName = nMap["NetworkAdapterName"].(string) + return nil +} + +func (ncfg *networkConfiguration) Key() []string { + return []string{windowsPrefix + ncfg.Type, ncfg.ID} +} + +func (ncfg *networkConfiguration) KeyPrefix() []string { + return []string{windowsPrefix + ncfg.Type} +} + +func (ncfg *networkConfiguration) Value() []byte { + b, err := json.Marshal(ncfg) + if err != nil { + return nil + } + return b +} + +func (ncfg *networkConfiguration) SetValue(value []byte) error { + return json.Unmarshal(value, ncfg) +} + +func (ncfg *networkConfiguration) Index() uint64 { + return ncfg.dbIndex +} + +func (ncfg *networkConfiguration) SetIndex(index uint64) { + ncfg.dbIndex = index + ncfg.dbExists = true +} + +func (ncfg *networkConfiguration) Exists() bool { + return ncfg.dbExists +} + +func (ncfg *networkConfiguration) Skip() bool { + return false +} + +func (ncfg *networkConfiguration) New() datastore.KVObject { + return &networkConfiguration{Type: ncfg.Type} +} + +func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error { + dstNcfg := o.(*networkConfiguration) + *dstNcfg = *ncfg + return nil +} + +func (ncfg *networkConfiguration) DataScope() string { + return datastore.LocalScope +} + +func (ep *hnsEndpoint) MarshalJSON() ([]byte, error) { + epMap := make(map[string]interface{}) + epMap["id"] = ep.id + epMap["nid"] = ep.nid + epMap["Type"] = ep.Type + epMap["profileID"] = ep.profileID + epMap["MacAddress"] = ep.macAddress.String() + epMap["Addr"] = ep.addr.String() + epMap["gateway"] = ep.gateway.String() + + epMap["epOption"] = ep.epOption + epMap["epConnectivity"] = ep.epConnectivity + epMap["PortMapping"] = ep.portMapping + + return json.Marshal(epMap) +} + +func (ep *hnsEndpoint) UnmarshalJSON(b []byte) error { + var ( + err error + epMap map[string]interface{} + ) + + if err = json.Unmarshal(b, &epMap); err != nil { + return fmt.Errorf("Failed to unmarshal to endpoint: %v", err) + } + + if v, ok := epMap["MacAddress"]; ok { + if ep.macAddress, err = net.ParseMAC(v.(string)); err != nil { + return types.InternalErrorf("failed to decode endpoint MAC address (%s) after json unmarshal: %v", v.(string), err) + } + } + if v, ok := epMap["Addr"]; ok { + if ep.addr, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err) + } + } + + ep.id = epMap["id"].(string) + ep.Type = epMap["Type"].(string) + ep.nid = epMap["nid"].(string) + ep.profileID = epMap["profileID"].(string) + d, _ := json.Marshal(epMap["epOption"]) + if err := json.Unmarshal(d, &ep.epOption); err != nil { + logrus.Warnf("Failed to decode endpoint container config %v", err) + } + d, _ = json.Marshal(epMap["epConnectivity"]) + if err := json.Unmarshal(d, &ep.epConnectivity); err != nil { + logrus.Warnf("Failed to decode endpoint external connectivity configuration %v", err) + } + d, _ = json.Marshal(epMap["PortMapping"]) + if err := json.Unmarshal(d, &ep.portMapping); err != nil { + logrus.Warnf("Failed to decode endpoint port mapping %v", err) + } + + return nil +} + +func (ep *hnsEndpoint) Key() []string { + return []string{windowsEndpointPrefix + ep.Type, ep.id} +} + +func (ep *hnsEndpoint) KeyPrefix() []string { + return []string{windowsEndpointPrefix + ep.Type} +} + +func (ep *hnsEndpoint) Value() []byte { + b, err := json.Marshal(ep) + if err != nil { + return nil + } + return b +} + +func (ep *hnsEndpoint) SetValue(value []byte) error { + return json.Unmarshal(value, ep) +} + +func (ep *hnsEndpoint) Index() uint64 { + return ep.dbIndex +} + +func (ep *hnsEndpoint) SetIndex(index uint64) { + ep.dbIndex = index + ep.dbExists = true +} + +func (ep *hnsEndpoint) Exists() bool { + return ep.dbExists +} + +func (ep *hnsEndpoint) Skip() bool { + return false +} + +func (ep *hnsEndpoint) New() datastore.KVObject { + return &hnsEndpoint{Type: ep.Type} +} + +func (ep *hnsEndpoint) CopyTo(o datastore.KVObject) error { + dstEp := o.(*hnsEndpoint) + *dstEp = *ep + return nil +} + +func (ep *hnsEndpoint) DataScope() string { + return datastore.LocalScope +}