diff --git a/common/setmatrix.go b/common/setmatrix.go index e0e6cea9bf..72be5bbbfc 100644 --- a/common/setmatrix.go +++ b/common/setmatrix.go @@ -28,6 +28,8 @@ type SetMatrix interface { // String returns the string version of the set, empty otherwise // returns false if the set is not present String(key string) (string, bool) + // Returns all the keys in the map + Keys() []string } type setMatrix struct { @@ -121,3 +123,13 @@ func (s *setMatrix) String(key string) (string, bool) { } return set.String(), ok } + +func (s *setMatrix) Keys() []string { + s.Lock() + defer s.Unlock() + keys := make([]string, 0, len(s.matrix)) + for k := range s.matrix { + keys = append(keys, k) + } + return keys +} diff --git a/libnetwork_internal_test.go b/libnetwork_internal_test.go index f0d803aa75..c5b8ae2ae7 100644 --- a/libnetwork_internal_test.go +++ b/libnetwork_internal_test.go @@ -13,6 +13,7 @@ import ( "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/ipamapi" "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/netutils" "github.com/docker/libnetwork/testutils" "github.com/docker/libnetwork/types" ) @@ -379,8 +380,8 @@ func TestSRVServiceQuery(t *testing.T) { } sr := svcInfo{ - svcMap: make(map[string][]net.IP), - svcIPv6Map: make(map[string][]net.IP), + svcMap: common.NewSetMatrix(), + svcIPv6Map: common.NewSetMatrix(), ipMap: common.NewSetMatrix(), service: make(map[string][]servicePorts), } @@ -437,6 +438,119 @@ func TestSRVServiceQuery(t *testing.T) { } } +func TestServiceVIPReuse(t *testing.T) { + c, err := New() + if err != nil { + t.Fatal(err) + } + defer c.Stop() + + n, err := c.NewNetwork("bridge", "net1", "", nil) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := n.Delete(); err != nil { + t.Fatal(err) + } + }() + + ep, err := n.CreateEndpoint("testep") + if err != nil { + t.Fatal(err) + } + + sb, err := c.NewSandbox("c1") + if err != nil { + t.Fatal(err) + } + defer func() { + if err := sb.Delete(); err != nil { + t.Fatal(err) + } + }() + + err = ep.Join(sb) + if err != nil { + t.Fatal(err) + } + + // Add 2 services with same name but different service ID to share the same VIP + n.(*network).addSvcRecords("ep1", "service_test", "serviceID1", net.ParseIP("192.168.0.1"), net.IP{}, true, "test") + n.(*network).addSvcRecords("ep2", "service_test", "serviceID2", net.ParseIP("192.168.0.1"), net.IP{}, true, "test") + + ipToResolve := netutils.ReverseIP("192.168.0.1") + + ipList, _ := n.(*network).ResolveName("service_test", types.IPv4) + if len(ipList) == 0 { + t.Fatal("There must be the VIP") + } + if len(ipList) != 1 { + t.Fatal("It must return only 1 VIP") + } + if ipList[0].String() != "192.168.0.1" { + t.Fatal("The service VIP is 192.168.0.1") + } + name := n.(*network).ResolveIP(ipToResolve) + if name == "" { + t.Fatal("It must return a name") + } + if name != "service_test.net1" { + t.Fatalf("It must return the service_test.net1 != %s", name) + } + + // Delete service record for one of the services, the IP should remain because one service is still associated with it + n.(*network).deleteSvcRecords("ep1", "service_test", "serviceID1", net.ParseIP("192.168.0.1"), net.IP{}, true, "test") + ipList, _ = n.(*network).ResolveName("service_test", types.IPv4) + if len(ipList) == 0 { + t.Fatal("There must be the VIP") + } + if len(ipList) != 1 { + t.Fatal("It must return only 1 VIP") + } + if ipList[0].String() != "192.168.0.1" { + t.Fatal("The service VIP is 192.168.0.1") + } + name = n.(*network).ResolveIP(ipToResolve) + if name == "" { + t.Fatal("It must return a name") + } + if name != "service_test.net1" { + t.Fatalf("It must return the service_test.net1 != %s", name) + } + + // Delete again the service using the previous service ID, nothing should happen + n.(*network).deleteSvcRecords("ep2", "service_test", "serviceID1", net.ParseIP("192.168.0.1"), net.IP{}, true, "test") + ipList, _ = n.(*network).ResolveName("service_test", types.IPv4) + if len(ipList) == 0 { + t.Fatal("There must be the VIP") + } + if len(ipList) != 1 { + t.Fatal("It must return only 1 VIP") + } + if ipList[0].String() != "192.168.0.1" { + t.Fatal("The service VIP is 192.168.0.1") + } + name = n.(*network).ResolveIP(ipToResolve) + if name == "" { + t.Fatal("It must return a name") + } + if name != "service_test.net1" { + t.Fatalf("It must return the service_test.net1 != %s", name) + } + + // Delete now using the second service ID, now all the entries should be gone + n.(*network).deleteSvcRecords("ep2", "service_test", "serviceID2", net.ParseIP("192.168.0.1"), net.IP{}, true, "test") + ipList, _ = n.(*network).ResolveName("service_test", types.IPv4) + if len(ipList) != 0 { + t.Fatal("All the VIPs should be gone now") + } + name = n.(*network).ResolveIP(ipToResolve) + if name != "" { + t.Fatalf("It must return empty no more services associated, instead:%s", name) + } +} + func TestIpamReleaseOnNetDriverFailures(t *testing.T) { if !testutils.IsRunningInContainer() { defer testutils.SetupTestOSContext(t)() diff --git a/network.go b/network.go index 2fddb59b54..5854a16820 100644 --- a/network.go +++ b/network.go @@ -81,9 +81,23 @@ type NetworkInfo interface { // When the function returns true, the walk will stop. type EndpointWalker func(ep Endpoint) bool +// ipInfo is the reverse mapping from IP to service name to serve the PTR query. +// Its an indication to defer PTR queries also to that external server. +type ipInfo struct { + name string + serviceID string +} + +// svcMapEntry is the body of the element into the svcMap +// The ip is a string because the SetMatrix does not accept non hashable values +type svcMapEntry struct { + ip string + serviceID string +} + type svcInfo struct { - svcMap map[string][]net.IP - svcIPv6Map map[string][]net.IP + svcMap common.SetMatrix + svcIPv6Map common.SetMatrix ipMap common.SetMatrix service map[string][]servicePorts } @@ -1047,73 +1061,76 @@ func (n *network) updateSvcRecord(ep *endpoint, localEps []*endpoint, isAdd bool ipv6 = iface.AddressIPv6().IP } + serviceID := ep.svcID if isAdd { // If anonymous endpoint has an alias use the first alias // for ip->name mapping. Not having the reverse mapping // breaks some apps if ep.isAnonymous() { + serviceID = ep.ID() if len(myAliases) > 0 { - n.addSvcRecords(ep.ID(), myAliases[0], iface.Address().IP, ipv6, true, "updateSvcRecord") + n.addSvcRecords(ep.ID(), myAliases[0], serviceID, iface.Address().IP, ipv6, true, "updateSvcRecord") } } else { - n.addSvcRecords(ep.ID(), epName, iface.Address().IP, ipv6, true, "updateSvcRecord") + n.addSvcRecords(ep.ID(), epName, serviceID, iface.Address().IP, ipv6, true, "updateSvcRecord") } for _, alias := range myAliases { - n.addSvcRecords(ep.ID(), alias, iface.Address().IP, ipv6, false, "updateSvcRecord") + n.addSvcRecords(ep.ID(), alias, serviceID, iface.Address().IP, ipv6, false, "updateSvcRecord") } } else { if ep.isAnonymous() { + serviceID = ep.ID() if len(myAliases) > 0 { - n.deleteSvcRecords(ep.ID(), myAliases[0], iface.Address().IP, ipv6, true, "updateSvcRecord") + n.deleteSvcRecords(ep.ID(), myAliases[0], serviceID, iface.Address().IP, ipv6, true, "updateSvcRecord") } } else { - n.deleteSvcRecords(ep.ID(), epName, iface.Address().IP, ipv6, true, "updateSvcRecord") + n.deleteSvcRecords(ep.ID(), epName, serviceID, iface.Address().IP, ipv6, true, "updateSvcRecord") } for _, alias := range myAliases { - n.deleteSvcRecords(ep.ID(), alias, iface.Address().IP, ipv6, false, "updateSvcRecord") + n.deleteSvcRecords(ep.ID(), alias, serviceID, iface.Address().IP, ipv6, false, "updateSvcRecord") } } } } -func addIPToName(ipMap common.SetMatrix, name string, ip net.IP) { +func addIPToName(ipMap common.SetMatrix, name, serviceID string, ip net.IP) { reverseIP := netutils.ReverseIP(ip.String()) - ipMap.Insert(reverseIP, name) + ipMap.Insert(reverseIP, ipInfo{ + name: name, + serviceID: serviceID, + }) } -func addNameToIP(svcMap map[string][]net.IP, name string, epIP net.IP) { - ipList := svcMap[name] - for _, ip := range ipList { - if ip.Equal(epIP) { - return - } - } - svcMap[name] = append(svcMap[name], epIP) +func delIPToName(ipMap common.SetMatrix, name, serviceID string, ip net.IP) { + reverseIP := netutils.ReverseIP(ip.String()) + ipMap.Remove(reverseIP, ipInfo{ + name: name, + serviceID: serviceID, + }) } -func delNameToIP(svcMap map[string][]net.IP, name string, epIP net.IP) { - ipList := svcMap[name] - for i, ip := range ipList { - if ip.Equal(epIP) { - ipList = append(ipList[:i], ipList[i+1:]...) - break - } - } - svcMap[name] = ipList +func addNameToIP(svcMap common.SetMatrix, name, serviceID string, epIP net.IP) { + svcMap.Insert(name, svcMapEntry{ + ip: epIP.String(), + serviceID: serviceID, + }) +} - if len(ipList) == 0 { - delete(svcMap, name) - } +func delNameToIP(svcMap common.SetMatrix, name, serviceID string, epIP net.IP) { + svcMap.Remove(name, svcMapEntry{ + ip: epIP.String(), + serviceID: serviceID, + }) } -func (n *network) addSvcRecords(eID, name string, epIP net.IP, epIPv6 net.IP, ipMapUpdate bool, method string) { +func (n *network) addSvcRecords(eID, name, serviceID string, epIP, epIPv6 net.IP, ipMapUpdate bool, method string) { // Do not add service names for ingress network as this is a // routing only network if n.ingress { return } - logrus.Debugf("%s (%s).addSvcRecords(%s, %s, %s, %t) %s", eID, n.ID()[0:7], name, epIP, epIPv6, ipMapUpdate, method) + logrus.Debugf("%s (%s).addSvcRecords(%s, %s, %s, %t) %s sid:%s", eID, n.ID()[0:7], name, epIP, epIPv6, ipMapUpdate, method, serviceID) c := n.getController() c.Lock() @@ -1122,34 +1139,34 @@ func (n *network) addSvcRecords(eID, name string, epIP net.IP, epIPv6 net.IP, ip sr, ok := c.svcRecords[n.ID()] if !ok { sr = svcInfo{ - svcMap: make(map[string][]net.IP), - svcIPv6Map: make(map[string][]net.IP), + svcMap: common.NewSetMatrix(), + svcIPv6Map: common.NewSetMatrix(), ipMap: common.NewSetMatrix(), } c.svcRecords[n.ID()] = sr } if ipMapUpdate { - addIPToName(sr.ipMap, name, epIP) + addIPToName(sr.ipMap, name, serviceID, epIP) if epIPv6 != nil { - addIPToName(sr.ipMap, name, epIPv6) + addIPToName(sr.ipMap, name, serviceID, epIPv6) } } - addNameToIP(sr.svcMap, name, epIP) + addNameToIP(sr.svcMap, name, serviceID, epIP) if epIPv6 != nil { - addNameToIP(sr.svcIPv6Map, name, epIPv6) + addNameToIP(sr.svcIPv6Map, name, serviceID, epIPv6) } } -func (n *network) deleteSvcRecords(eID, name string, epIP net.IP, epIPv6 net.IP, ipMapUpdate bool, method string) { +func (n *network) deleteSvcRecords(eID, name, serviceID string, epIP net.IP, epIPv6 net.IP, ipMapUpdate bool, method string) { // Do not delete service names from ingress network as this is a // routing only network if n.ingress { return } - logrus.Debugf("%s (%s).deleteSvcRecords(%s, %s, %s, %t) %s", eID, n.ID()[0:7], name, epIP, epIPv6, ipMapUpdate, method) + logrus.Debugf("%s (%s).deleteSvcRecords(%s, %s, %s, %t) %s sid:%s ", eID, n.ID()[0:7], name, epIP, epIPv6, ipMapUpdate, method, serviceID) c := n.getController() c.Lock() @@ -1161,17 +1178,17 @@ func (n *network) deleteSvcRecords(eID, name string, epIP net.IP, epIPv6 net.IP, } if ipMapUpdate { - sr.ipMap.Remove(netutils.ReverseIP(epIP.String()), name) + delIPToName(sr.ipMap, name, serviceID, epIP) if epIPv6 != nil { - sr.ipMap.Remove(netutils.ReverseIP(epIPv6.String()), name) + delIPToName(sr.ipMap, name, serviceID, epIPv6) } } - delNameToIP(sr.svcMap, name, epIP) + delNameToIP(sr.svcMap, name, serviceID, epIP) if epIPv6 != nil { - delNameToIP(sr.svcIPv6Map, name, epIPv6) + delNameToIP(sr.svcIPv6Map, name, serviceID, epIPv6) } } @@ -1189,19 +1206,31 @@ func (n *network) getSvcRecords(ep *endpoint) []etchosts.Record { n.ctrlr.Lock() defer n.ctrlr.Unlock() - sr, _ := n.ctrlr.svcRecords[n.id] + sr, ok := n.ctrlr.svcRecords[n.id] + if !ok || sr.svcMap == nil { + return nil + } - for h, ip := range sr.svcMap { - if strings.Split(h, ".")[0] == epName { + svcMapKeys := sr.svcMap.Keys() + // Loop on service names on this network + for _, k := range svcMapKeys { + if strings.Split(k, ".")[0] == epName { + continue + } + // Get all the IPs associated to this service + mapEntryList, ok := sr.svcMap.Get(k) + if !ok { + // The key got deleted continue } - if len(ip) == 0 { - logrus.Warnf("Found empty list of IP addresses for service %s on network %s (%s)", h, n.name, n.id) + if len(mapEntryList) == 0 { + logrus.Warnf("Found empty list of IP addresses for service %s on network %s (%s)", k, n.name, n.id) continue } + recs = append(recs, etchosts.Record{ - Hosts: h, - IP: ip[0].String(), + Hosts: k, + IP: mapEntryList[0].(svcMapEntry).ip, }) } @@ -1622,17 +1651,15 @@ func (n *network) ResolveName(req string, ipType int) ([]net.IP, bool) { c := n.getController() c.Lock() + defer c.Unlock() sr, ok := c.svcRecords[n.ID()] - c.Unlock() if !ok { return nil, false } req = strings.TrimSuffix(req, ".") - var ip []net.IP - n.Lock() - ip, ok = sr.svcMap[req] + ipSet, ok := sr.svcMap.Get(req) if ipType == types.IPv6 { // If the name resolved to v4 address then its a valid name in @@ -1642,14 +1669,20 @@ func (n *network) ResolveName(req string, ipType int) ([]net.IP, bool) { if ok && n.enableIPv6 == false { ipv6Miss = true } - ip = sr.svcIPv6Map[req] + ipSet, ok = sr.svcIPv6Map.Get(req) } - n.Unlock() - if ip != nil { - ipLocal := make([]net.IP, len(ip)) - copy(ipLocal, ip) - return ipLocal, false + if ok && len(ipSet) > 0 { + // this maps is to avoid IP duplicates, this can happen during a transition period where 2 services are using the same IP + noDup := make(map[string]bool) + var ipLocal []net.IP + for _, ip := range ipSet { + if _, dup := noDup[ip.(svcMapEntry).ip]; !dup { + noDup[ip.(svcMapEntry).ip] = true + ipLocal = append(ipLocal, net.ParseIP(ip.(svcMapEntry).ip)) + } + } + return ipLocal, ok } return nil, ipv6Miss @@ -1658,8 +1691,8 @@ func (n *network) ResolveName(req string, ipType int) ([]net.IP, bool) { func (n *network) ResolveIP(ip string) string { c := n.getController() c.Lock() + defer c.Unlock() sr, ok := c.svcRecords[n.ID()] - c.Unlock() if !ok { return "" @@ -1667,23 +1700,23 @@ func (n *network) ResolveIP(ip string) string { nwName := n.Name() - nameList, ok := sr.ipMap.Get(ip) - if !ok || len(nameList) == 0 { + elemSet, ok := sr.ipMap.Get(ip) + if !ok || len(elemSet) == 0 { return "" } // NOTE it is possible to have more than one element in the Set, this will happen - // because of interleave of diffent events from differnt sources (local container create vs + // because of interleave of different events from different sources (local container create vs // network db notifications) // In such cases the resolution will be based on the first element of the set, and can vary // during the system stabilitation - name, ok := nameList[0].(string) + elem, ok := elemSet[0].(ipInfo) if !ok { setStr, b := sr.ipMap.String(ip) - logrus.Errorf("expected set of strings for key %s set:%t %s", ip, b, setStr) + logrus.Errorf("expected set of ipInfo type for key %s set:%t %s", ip, b, setStr) return "" } - return name + "." + nwName + return elem.name + "." + nwName } func (n *network) ResolveService(name string) ([]*net.SRV, []net.IP) { diff --git a/service_common.go b/service_common.go index 31a3fd9283..e33242a63d 100644 --- a/service_common.go +++ b/service_common.go @@ -21,23 +21,23 @@ func (c *controller) addEndpointNameResolution(svcName, svcID, nID, eID, contain c.addContainerNameResolution(nID, eID, containerName, taskAliases, ip, method) // Add endpoint IP to special "tasks.svc_name" so that the applications have access to DNS RR. - n.(*network).addSvcRecords(eID, "tasks."+svcName, ip, nil, false, method) + n.(*network).addSvcRecords(eID, "tasks."+svcName, svcID, ip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).addSvcRecords(eID, "tasks."+alias, ip, nil, false, method) + n.(*network).addSvcRecords(eID, "tasks."+alias, svcID, ip, nil, false, method) } // Add service name to vip in DNS, if vip is valid. Otherwise resort to DNS RR if len(vip) == 0 { - n.(*network).addSvcRecords(eID, svcName, ip, nil, false, method) + n.(*network).addSvcRecords(eID, svcName, eID, ip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).addSvcRecords(eID, alias, ip, nil, false, method) + n.(*network).addSvcRecords(eID, alias, eID, ip, nil, false, method) } } if addService && len(vip) != 0 { - n.(*network).addSvcRecords(eID, svcName, vip, nil, false, method) + n.(*network).addSvcRecords(eID, svcName, svcID, vip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).addSvcRecords(eID, alias, vip, nil, false, method) + n.(*network).addSvcRecords(eID, alias, svcID, vip, nil, false, method) } } @@ -52,11 +52,11 @@ func (c *controller) addContainerNameResolution(nID, eID, containerName string, logrus.Debugf("addContainerNameResolution %s %s", eID, containerName) // Add resolution for container name - n.(*network).addSvcRecords(eID, containerName, ip, nil, true, method) + n.(*network).addSvcRecords(eID, containerName, eID, ip, nil, true, method) // Add resolution for taskaliases for _, alias := range taskAliases { - n.(*network).addSvcRecords(eID, alias, ip, nil, true, method) + n.(*network).addSvcRecords(eID, alias, eID, ip, nil, true, method) } return nil @@ -75,25 +75,25 @@ func (c *controller) deleteEndpointNameResolution(svcName, svcID, nID, eID, cont // Delete the special "tasks.svc_name" backend record. if !multipleEntries { - n.(*network).deleteSvcRecords(eID, "tasks."+svcName, ip, nil, false, method) + n.(*network).deleteSvcRecords(eID, "tasks."+svcName, svcID, ip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).deleteSvcRecords(eID, "tasks."+alias, ip, nil, false, method) + n.(*network).deleteSvcRecords(eID, "tasks."+alias, svcID, ip, nil, false, method) } } // If we are doing DNS RR delete the endpoint IP from DNS record right away. if !multipleEntries && len(vip) == 0 { - n.(*network).deleteSvcRecords(eID, svcName, ip, nil, false, method) + n.(*network).deleteSvcRecords(eID, svcName, eID, ip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).deleteSvcRecords(eID, alias, ip, nil, false, method) + n.(*network).deleteSvcRecords(eID, alias, eID, ip, nil, false, method) } } // Remove the DNS record for VIP only if we are removing the service if rmService && len(vip) != 0 && !multipleEntries { - n.(*network).deleteSvcRecords(eID, svcName, vip, nil, false, method) + n.(*network).deleteSvcRecords(eID, svcName, svcID, vip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).deleteSvcRecords(eID, alias, vip, nil, false, method) + n.(*network).deleteSvcRecords(eID, alias, svcID, vip, nil, false, method) } } @@ -108,11 +108,11 @@ func (c *controller) delContainerNameResolution(nID, eID, containerName string, logrus.Debugf("delContainerNameResolution %s %s", eID, containerName) // Delete resolution for container name - n.(*network).deleteSvcRecords(eID, containerName, ip, nil, true, method) + n.(*network).deleteSvcRecords(eID, containerName, eID, ip, nil, true, method) // Delete resolution for taskaliases for _, alias := range taskAliases { - n.(*network).deleteSvcRecords(eID, alias, ip, nil, true, method) + n.(*network).deleteSvcRecords(eID, alias, eID, ip, nil, true, method) } return nil @@ -208,8 +208,7 @@ func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName s } s.Unlock() } - logrus.Debugf("addServiceBinding from %s START for %s %s", method, svcName, eID) - + logrus.Debugf("addServiceBinding from %s START for %s %s p:%p nid:%s skey:%v", method, svcName, eID, s, nID, skey) defer s.Unlock() lb, ok := s.loadBalancers[nID] @@ -274,7 +273,6 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st c.Lock() s, ok := c.serviceBindings[skey] c.Unlock() - logrus.Debugf("rmServiceBinding from %s START for %s %s", method, svcName, eID) if !ok { logrus.Warnf("rmServiceBinding %s %s %s aborted c.serviceBindings[skey] !ok", method, svcName, eID) return nil @@ -282,6 +280,7 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st s.Lock() defer s.Unlock() + logrus.Debugf("rmServiceBinding from %s START for %s %s p:%p nid:%s sKey:%v", method, svcName, eID, s, nID, skey) lb, ok := s.loadBalancers[nID] if !ok { logrus.Warnf("rmServiceBinding %s %s %s aborted s.loadBalancers[nid] !ok", method, svcName, eID) @@ -302,17 +301,7 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st rmService = true delete(s.loadBalancers, nID) - } - - if len(s.loadBalancers) == 0 { - // All loadbalancers for the service removed. Time to - // remove the service itself. - c.Lock() - - // Mark the object as deleted so that the add won't use it wrongly - s.deleted = true - delete(c.serviceBindings, skey) - c.Unlock() + logrus.Debugf("rmServiceBinding %s delete %s, p:%p in loadbalancers len:%d", eID, nID, lb, len(s.loadBalancers)) } ok, entries := s.removeIPToEndpoint(ip.String(), eID) @@ -330,6 +319,19 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st // Delete the name resolutions c.deleteEndpointNameResolution(svcName, svcID, nID, eID, containerName, vip, serviceAliases, taskAliases, ip, rmService, entries > 0, "rmServiceBinding") + if len(s.loadBalancers) == 0 { + // All loadbalancers for the service removed. Time to + // remove the service itself. + c.Lock() + + // Mark the object as deleted so that the add won't use it wrongly + s.deleted = true + // NOTE The delete from the serviceBindings map has to be the last operation else we are allowing a race between this service + // that is getting deleted and a new service that will be created if the entry is not anymore there + delete(c.serviceBindings, skey) + c.Unlock() + } + logrus.Debugf("rmServiceBinding from %s END for %s %s", method, svcName, eID) return nil }