Merge pull request gliderlabs#13 from auto1-oss/do-not-add-killed-con…

…tainers

OPS-13251 Added skipping logic for killed containers

bridge/bridge.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	dockerapi "github.com/fsouza/go-dockerclient"
 )
@@ -19,11 +20,12 @@ var serviceIDPattern = regexp.MustCompile(`^(.+?):([a-zA-Z0-9][a-zA-Z0-9_.-]+):[
 
 type Bridge struct {
 	sync.Mutex
-	registry       RegistryAdapter
-	docker         *dockerapi.Client
-	services       map[string][]*Service
-	deadContainers map[string]*DeadContainer
-	config         Config
+	registry        RegistryAdapter
+	docker          *dockerapi.Client
+	services        map[string][]*Service
+	deadContainers  map[string]*DeadContainer
+	dyingContainers map[string]time.Time
+	config          Config
 }
 
 func New(docker *dockerapi.Client, adapterUri string, config Config) (*Bridge, error) {
@@ -38,11 +40,12 @@ func New(docker *dockerapi.Client, adapterUri string, config Config) (*Bridge, e
 
 	log.Println("Using", uri.Scheme, "adapter:", uri)
 	return &Bridge{
-		docker:         docker,
-		config:         config,
-		registry:       factory.New(uri),
-		services:       make(map[string][]*Service),
-		deadContainers: make(map[string]*DeadContainer),
+		docker:          docker,
+		config:          config,
+		registry:        factory.New(uri),
+		services:        make(map[string][]*Service),
+		deadContainers:  make(map[string]*DeadContainer),
+		dyingContainers: make(map[string]time.Time),
 	}, nil
 }
 
@@ -187,6 +190,11 @@ func (b *Bridge) Sync(quiet bool) {
 }
 
 func (b *Bridge) add(containerId string, quiet bool) {
+	if _, ok := b.dyingContainers[containerId]; ok {
+		log.Println("container, ", containerId[:12], ", is dying, ignoring")
+		return
+	}
+
 	if d := b.deadContainers[containerId]; d != nil {
 		b.services[containerId] = d.Services
 		delete(b.deadContainers, containerId)
@@ -394,6 +402,7 @@ func (b *Bridge) remove(containerId string, deregister bool) {
 		b.deadContainers[containerId] = &DeadContainer{b.config.RefreshTtl, b.services[containerId]}
 	}
 	delete(b.services, containerId)
+	b.markContainerAsDying(containerId)
 }
 
 // bit set on ExitCode if it represents an exit via a signal
@@ -427,6 +436,18 @@ func (b *Bridge) shouldRemove(containerId string) bool {
 	return false
 }
 
+func (b *Bridge) markContainerAsDying(containerId string) {
+	// cleanup after CleanupDyingTtl
+	for containerId, t := range b.dyingContainers {
+		if time.Since(t) >= time.Millisecond * time.Duration(b.config.CleanupDyingTtl) {
+			delete(b.dyingContainers, containerId)
+		}
+	}
+
+	// mark container as "dying"
+	b.dyingContainers[containerId] = time.Now()
+}
+
 var Hostname string
 
 func init() {

bridge/types.go

@@ -30,6 +30,7 @@ type Config struct {
 	RefreshInterval int
 	DeregisterCheck string
 	Cleanup         bool
+	CleanupDyingTtl int
 }
 
 type Service struct {

registrator.go

@@ -32,6 +32,7 @@ var deregisterOnStop = flag.Bool("deregister-on-stop", false, "Deregister when c
 var retryAttempts = flag.Int("retry-attempts", 0, "Max retry attempts to establish a connection with the backend. Use -1 for infinite retries")
 var retryInterval = flag.Int("retry-interval", 2000, "Interval (in millisecond) between retry-attempts.")
 var cleanup = flag.Bool("cleanup", false, "Remove dangling services")
+var cleanupDyingTtl = flag.Int("ttl-dying-cleanup", 60000, "TTL (in millisecond) for cleaning dying containers cache")
 
 func getopt(name, def string) string {
 	if env := os.Getenv(name); env != "" {
@@ -113,6 +114,7 @@ func main() {
 		RefreshInterval: *refreshInterval,
 		DeregisterCheck: *deregister,
 		Cleanup:         *cleanup,
+		CleanupDyingTtl: *cleanupDyingTtl,
 	})
 
 	assert(err)