Added persistence to windows driver so that cleanup happens properly
Signed-off-by: Sandeep Bansal <sabansal@microsoft.com>
This commit is contained in:
Sandeep Bansal
parent
89b89862aa
commit
f566d6384e
2 changed files with 401 additions and 22 deletions
|
|
@ -41,6 +41,8 @@ type networkConfiguration struct {
|
|||
DNSSuffix string
|
||||
SourceMac string
|
||||
NetworkAdapterName string
|
||||
dbIndex uint64
|
||||
dbExists bool
|
||||
}
|
||||
|
||||
// endpointConfiguration represents the user specified configuration for the sandbox endpoint
|
||||
|
|
@ -59,17 +61,22 @@ type endpointConnectivity struct {
|
|||
|
||||
type hnsEndpoint struct {
|
||||
id string
|
||||
nid string
|
||||
profileID string
|
||||
Type string
|
||||
macAddress net.HardwareAddr
|
||||
epOption *endpointOption // User specified parameters
|
||||
epConnectivity *endpointConnectivity // User specified parameters
|
||||
portMapping []types.PortBinding // Operation port bindings
|
||||
addr *net.IPNet
|
||||
gateway net.IP
|
||||
dbIndex uint64
|
||||
dbExists bool
|
||||
}
|
||||
|
||||
type hnsNetwork struct {
|
||||
id string
|
||||
created bool
|
||||
config *networkConfiguration
|
||||
endpoints map[string]*hnsEndpoint // key: endpoint id
|
||||
driver *driver // The network's driver
|
||||
|
|
@ -79,9 +86,14 @@ type hnsNetwork struct {
|
|||
type driver struct {
|
||||
name string
|
||||
networks map[string]*hnsNetwork
|
||||
store datastore.DataStore
|
||||
sync.Mutex
|
||||
}
|
||||
|
||||
const (
|
||||
errNotFound = "HNS failed with error : The object identifier does not represent a valid object. "
|
||||
)
|
||||
|
||||
// IsBuiltinWindowsDriver vaidates if network-type is a builtin local-scoped driver
|
||||
func IsBuiltinLocalDriver(networkType string) bool {
|
||||
if "l2bridge" == networkType || "l2tunnel" == networkType || "nat" == networkType || "ics" == networkType || "transparent" == networkType {
|
||||
|
|
@ -103,7 +115,14 @@ func GetInit(networkType string) func(dc driverapi.DriverCallback, config map[st
|
|||
return types.BadRequestErrorf("Network type not supported: %s", networkType)
|
||||
}
|
||||
|
||||
return dc.RegisterDriver(networkType, newDriver(networkType), driverapi.Capability{
|
||||
d := newDriver(networkType)
|
||||
|
||||
err := d.initStore(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return dc.RegisterDriver(networkType, d, driverapi.Capability{
|
||||
DataScope: datastore.LocalScope,
|
||||
ConnectivityScope: datastore.LocalScope,
|
||||
})
|
||||
|
|
@ -133,7 +152,7 @@ func (n *hnsNetwork) getEndpoint(eid string) (*hnsEndpoint, error) {
|
|||
}
|
||||
|
||||
func (d *driver) parseNetworkOptions(id string, genericOptions map[string]string) (*networkConfiguration, error) {
|
||||
config := &networkConfiguration{}
|
||||
config := &networkConfiguration{Type: d.name}
|
||||
|
||||
for label, value := range genericOptions {
|
||||
switch label {
|
||||
|
|
@ -188,6 +207,21 @@ func (d *driver) DecodeTableEntry(tablename string, key string, value []byte) (s
|
|||
return "", nil
|
||||
}
|
||||
|
||||
func (d *driver) createNetwork(config *networkConfiguration) error {
|
||||
network := &hnsNetwork{
|
||||
id: config.ID,
|
||||
endpoints: make(map[string]*hnsEndpoint),
|
||||
config: config,
|
||||
driver: d,
|
||||
}
|
||||
|
||||
d.Lock()
|
||||
d.networks[config.ID] = network
|
||||
d.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create a new network
|
||||
func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
|
||||
if _, err := d.getNetwork(id); err == nil {
|
||||
|
|
@ -210,16 +244,11 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d
|
|||
return err
|
||||
}
|
||||
|
||||
network := &hnsNetwork{
|
||||
id: config.ID,
|
||||
endpoints: make(map[string]*hnsEndpoint),
|
||||
config: config,
|
||||
driver: d,
|
||||
}
|
||||
err = d.createNetwork(config)
|
||||
|
||||
d.Lock()
|
||||
d.networks[config.ID] = network
|
||||
d.Unlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// A non blank hnsid indicates that the network was discovered
|
||||
// from HNS. No need to call HNS if this network was discovered
|
||||
|
|
@ -294,7 +323,9 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d
|
|||
genData[HNSID] = config.HnsID
|
||||
}
|
||||
|
||||
return nil
|
||||
n, err := d.getNetwork(id)
|
||||
n.created = true
|
||||
return d.storeUpdate(config)
|
||||
}
|
||||
|
||||
func (d *driver) DeleteNetwork(nid string) error {
|
||||
|
|
@ -307,21 +338,25 @@ func (d *driver) DeleteNetwork(nid string) error {
|
|||
config := n.config
|
||||
n.Unlock()
|
||||
|
||||
// Cannot remove network if endpoints are still present
|
||||
if len(n.endpoints) != 0 {
|
||||
return fmt.Errorf("network %s has active endpoint", n.id)
|
||||
}
|
||||
|
||||
_, err = hcsshim.HNSNetworkRequest("DELETE", config.HnsID, "")
|
||||
if err != nil {
|
||||
return types.ForbiddenErrorf(err.Error())
|
||||
if n.created {
|
||||
_, err = hcsshim.HNSNetworkRequest("DELETE", config.HnsID, "")
|
||||
if err != nil && err.Error() != errNotFound {
|
||||
return types.ForbiddenErrorf(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
d.Lock()
|
||||
delete(d.networks, nid)
|
||||
d.Unlock()
|
||||
|
||||
return nil
|
||||
// delele endpoints belong to this network
|
||||
for _, ep := range n.endpoints {
|
||||
if err := d.storeDelete(ep); err != nil {
|
||||
logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
|
||||
}
|
||||
}
|
||||
|
||||
return d.storeDelete(config)
|
||||
}
|
||||
|
||||
func convertQosPolicies(qosPolicies []types.QosPolicy) ([]json.RawMessage, error) {
|
||||
|
|
@ -544,6 +579,8 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
|
|||
// TODO For now the ip mask is not in the info generated by HNS
|
||||
endpoint := &hnsEndpoint{
|
||||
id: eid,
|
||||
nid: n.id,
|
||||
Type: d.name,
|
||||
addr: &net.IPNet{IP: hnsresponse.IPAddress, Mask: hnsresponse.IPAddress.DefaultMask()},
|
||||
macAddress: mac,
|
||||
}
|
||||
|
|
@ -574,6 +611,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
|
|||
ifInfo.SetMacAddress(endpoint.macAddress)
|
||||
}
|
||||
|
||||
if err = d.storeUpdate(endpoint); err != nil {
|
||||
return fmt.Errorf("failed to save endpoint %s to store: %v", endpoint.id[0:7], err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -593,10 +634,13 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
|
|||
n.Unlock()
|
||||
|
||||
_, err = hcsshim.HNSEndpointRequest("DELETE", ep.profileID, "")
|
||||
if err != nil {
|
||||
if err != nil && err.Error() != errNotFound {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := d.storeDelete(ep); err != nil {
|
||||
logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
335
libnetwork/drivers/windows/windows_store.go
Normal file
335
libnetwork/drivers/windows/windows_store.go
Normal file
|
|
@ -0,0 +1,335 @@
|
|||
// +build windows
|
||||
|
||||
package windows
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/docker/libnetwork/datastore"
|
||||
"github.com/docker/libnetwork/discoverapi"
|
||||
"github.com/docker/libnetwork/netlabel"
|
||||
"github.com/docker/libnetwork/types"
|
||||
)
|
||||
|
||||
const (
|
||||
windowsPrefix = "windows"
|
||||
windowsEndpointPrefix = "windows-endpoint"
|
||||
)
|
||||
|
||||
func (d *driver) initStore(option map[string]interface{}) error {
|
||||
if data, ok := option[netlabel.LocalKVClient]; ok {
|
||||
var err error
|
||||
dsc, ok := data.(discoverapi.DatastoreConfigData)
|
||||
if !ok {
|
||||
return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
|
||||
}
|
||||
d.store, err = datastore.NewDataStoreFromConfig(dsc)
|
||||
if err != nil {
|
||||
return types.InternalErrorf("windows driver failed to initialize data store: %v", err)
|
||||
}
|
||||
|
||||
err = d.populateNetworks()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.populateEndpoints()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *driver) populateNetworks() error {
|
||||
kvol, err := d.store.List(datastore.Key(windowsPrefix), &networkConfiguration{Type: d.name})
|
||||
if err != nil && err != datastore.ErrKeyNotFound {
|
||||
return fmt.Errorf("failed to get windows network configurations from store: %v", err)
|
||||
}
|
||||
|
||||
// It's normal for network configuration state to be empty. Just return.
|
||||
if err == datastore.ErrKeyNotFound {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, kvo := range kvol {
|
||||
ncfg := kvo.(*networkConfiguration)
|
||||
if ncfg.Type != d.name {
|
||||
continue
|
||||
}
|
||||
if err = d.createNetwork(ncfg); err != nil {
|
||||
logrus.Warnf("could not create windows network for id %s hnsid %s while booting up from persistent state: %v", ncfg.ID, ncfg.HnsID, err)
|
||||
}
|
||||
logrus.Debugf("Network (%s) restored", ncfg.ID[0:7])
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *driver) populateEndpoints() error {
|
||||
kvol, err := d.store.List(datastore.Key(windowsEndpointPrefix), &hnsEndpoint{Type: d.name})
|
||||
if err != nil && err != datastore.ErrKeyNotFound {
|
||||
return fmt.Errorf("failed to get endpoints from store: %v", err)
|
||||
}
|
||||
|
||||
if err == datastore.ErrKeyNotFound {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, kvo := range kvol {
|
||||
ep := kvo.(*hnsEndpoint)
|
||||
if ep.Type != d.name {
|
||||
continue
|
||||
}
|
||||
n, ok := d.networks[ep.nid]
|
||||
if !ok {
|
||||
logrus.Debugf("Network (%s) not found for restored endpoint (%s)", ep.nid[0:7], ep.id[0:7])
|
||||
logrus.Debugf("Deleting stale endpoint (%s) from store", ep.id[0:7])
|
||||
if err := d.storeDelete(ep); err != nil {
|
||||
logrus.Debugf("Failed to delete stale endpoint (%s) from store", ep.id[0:7])
|
||||
}
|
||||
continue
|
||||
}
|
||||
n.endpoints[ep.id] = ep
|
||||
logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *driver) storeUpdate(kvObject datastore.KVObject) error {
|
||||
if d.store == nil {
|
||||
logrus.Warnf("store not initialized. kv object %s is not added to the store", datastore.Key(kvObject.Key()...))
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := d.store.PutObjectAtomic(kvObject); err != nil {
|
||||
return fmt.Errorf("failed to update store for object type %T: %v", kvObject, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *driver) storeDelete(kvObject datastore.KVObject) error {
|
||||
if d.store == nil {
|
||||
logrus.Debugf("store not initialized. kv object %s is not deleted from store", datastore.Key(kvObject.Key()...))
|
||||
return nil
|
||||
}
|
||||
|
||||
retry:
|
||||
if err := d.store.DeleteObjectAtomic(kvObject); err != nil {
|
||||
if err == datastore.ErrKeyModified {
|
||||
if err := d.store.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil {
|
||||
return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err)
|
||||
}
|
||||
goto retry
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) MarshalJSON() ([]byte, error) {
|
||||
nMap := make(map[string]interface{})
|
||||
|
||||
nMap["ID"] = ncfg.ID
|
||||
nMap["Type"] = ncfg.Type
|
||||
nMap["Name"] = ncfg.Name
|
||||
nMap["HnsID"] = ncfg.HnsID
|
||||
nMap["VLAN"] = ncfg.VLAN
|
||||
nMap["VSID"] = ncfg.VSID
|
||||
nMap["DNSServers"] = ncfg.DNSServers
|
||||
nMap["DNSSuffix"] = ncfg.DNSSuffix
|
||||
nMap["SourceMac"] = ncfg.SourceMac
|
||||
nMap["NetworkAdapterName"] = ncfg.NetworkAdapterName
|
||||
|
||||
return json.Marshal(nMap)
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) UnmarshalJSON(b []byte) error {
|
||||
var (
|
||||
err error
|
||||
nMap map[string]interface{}
|
||||
)
|
||||
|
||||
if err = json.Unmarshal(b, &nMap); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ncfg.ID = nMap["ID"].(string)
|
||||
ncfg.Type = nMap["Type"].(string)
|
||||
ncfg.Name = nMap["Name"].(string)
|
||||
ncfg.HnsID = nMap["HnsID"].(string)
|
||||
ncfg.VLAN = uint(nMap["VLAN"].(float64))
|
||||
ncfg.VSID = uint(nMap["VSID"].(float64))
|
||||
ncfg.DNSServers = nMap["DNSServers"].(string)
|
||||
ncfg.DNSSuffix = nMap["DNSSuffix"].(string)
|
||||
ncfg.SourceMac = nMap["SourceMac"].(string)
|
||||
ncfg.NetworkAdapterName = nMap["NetworkAdapterName"].(string)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) Key() []string {
|
||||
return []string{windowsPrefix + ncfg.Type, ncfg.ID}
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) KeyPrefix() []string {
|
||||
return []string{windowsPrefix + ncfg.Type}
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) Value() []byte {
|
||||
b, err := json.Marshal(ncfg)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) SetValue(value []byte) error {
|
||||
return json.Unmarshal(value, ncfg)
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) Index() uint64 {
|
||||
return ncfg.dbIndex
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) SetIndex(index uint64) {
|
||||
ncfg.dbIndex = index
|
||||
ncfg.dbExists = true
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) Exists() bool {
|
||||
return ncfg.dbExists
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) Skip() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) New() datastore.KVObject {
|
||||
return &networkConfiguration{Type: ncfg.Type}
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error {
|
||||
dstNcfg := o.(*networkConfiguration)
|
||||
*dstNcfg = *ncfg
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ncfg *networkConfiguration) DataScope() string {
|
||||
return datastore.LocalScope
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) MarshalJSON() ([]byte, error) {
|
||||
epMap := make(map[string]interface{})
|
||||
epMap["id"] = ep.id
|
||||
epMap["nid"] = ep.nid
|
||||
epMap["Type"] = ep.Type
|
||||
epMap["profileID"] = ep.profileID
|
||||
epMap["MacAddress"] = ep.macAddress.String()
|
||||
epMap["Addr"] = ep.addr.String()
|
||||
epMap["gateway"] = ep.gateway.String()
|
||||
|
||||
epMap["epOption"] = ep.epOption
|
||||
epMap["epConnectivity"] = ep.epConnectivity
|
||||
epMap["PortMapping"] = ep.portMapping
|
||||
|
||||
return json.Marshal(epMap)
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) UnmarshalJSON(b []byte) error {
|
||||
var (
|
||||
err error
|
||||
epMap map[string]interface{}
|
||||
)
|
||||
|
||||
if err = json.Unmarshal(b, &epMap); err != nil {
|
||||
return fmt.Errorf("Failed to unmarshal to endpoint: %v", err)
|
||||
}
|
||||
|
||||
if v, ok := epMap["MacAddress"]; ok {
|
||||
if ep.macAddress, err = net.ParseMAC(v.(string)); err != nil {
|
||||
return types.InternalErrorf("failed to decode endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)
|
||||
}
|
||||
}
|
||||
if v, ok := epMap["Addr"]; ok {
|
||||
if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {
|
||||
return types.InternalErrorf("failed to decode endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)
|
||||
}
|
||||
}
|
||||
|
||||
ep.id = epMap["id"].(string)
|
||||
ep.Type = epMap["Type"].(string)
|
||||
ep.nid = epMap["nid"].(string)
|
||||
ep.profileID = epMap["profileID"].(string)
|
||||
d, _ := json.Marshal(epMap["epOption"])
|
||||
if err := json.Unmarshal(d, &ep.epOption); err != nil {
|
||||
logrus.Warnf("Failed to decode endpoint container config %v", err)
|
||||
}
|
||||
d, _ = json.Marshal(epMap["epConnectivity"])
|
||||
if err := json.Unmarshal(d, &ep.epConnectivity); err != nil {
|
||||
logrus.Warnf("Failed to decode endpoint external connectivity configuration %v", err)
|
||||
}
|
||||
d, _ = json.Marshal(epMap["PortMapping"])
|
||||
if err := json.Unmarshal(d, &ep.portMapping); err != nil {
|
||||
logrus.Warnf("Failed to decode endpoint port mapping %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) Key() []string {
|
||||
return []string{windowsEndpointPrefix + ep.Type, ep.id}
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) KeyPrefix() []string {
|
||||
return []string{windowsEndpointPrefix + ep.Type}
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) Value() []byte {
|
||||
b, err := json.Marshal(ep)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) SetValue(value []byte) error {
|
||||
return json.Unmarshal(value, ep)
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) Index() uint64 {
|
||||
return ep.dbIndex
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) SetIndex(index uint64) {
|
||||
ep.dbIndex = index
|
||||
ep.dbExists = true
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) Exists() bool {
|
||||
return ep.dbExists
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) Skip() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) New() datastore.KVObject {
|
||||
return &hnsEndpoint{Type: ep.Type}
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) CopyTo(o datastore.KVObject) error {
|
||||
dstEp := o.(*hnsEndpoint)
|
||||
*dstEp = *ep
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ep *hnsEndpoint) DataScope() string {
|
||||
return datastore.LocalScope
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue