From c9246d3d4c162e0f7fe76f2ee37c55bdbfa4b0c6 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Wed, 30 Jun 2021 20:25:52 -0700 Subject: [PATCH] fix(executor): Tolerate docker re-creating containers. Fixes #6244 (#6252) Signed-off-by: Alex Collins --- workflow/executor/docker/docker.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/workflow/executor/docker/docker.go b/workflow/executor/docker/docker.go index 6a6d0607ffb8..4f6a0659f3fb 100644 --- a/workflow/executor/docker/docker.go +++ b/workflow/executor/docker/docker.go @@ -198,20 +198,27 @@ func (d *DockerExecutor) Wait(ctx context.Context, containerNames []string) erro log.WithError(err).Error("failed to poll container IDs") } }() - for !d.haveContainers(containerNames) { + for { select { case <-ctx.Done(): return ctx.Err() default: - time.Sleep(1 * time.Second) + if d.haveContainers(containerNames) { + containerIDs, err := d.getContainerIDs(containerNames) + if err != nil { + return err + } + _, err = common.RunCommand("docker", append([]string{"wait"}, containerIDs...)...) + if err != nil && strings.Contains(err.Error(), "No such container") { + // e.g. reason could be ContainerCannotRun + log.WithError(err).Info("ignoring error as container may have been re-created and therefore container ID may have changed") + continue + } + return err + } + time.Sleep(time.Second) } } - containerIDs, err := d.getContainerIDs(containerNames) - if err != nil { - return err - } - _, err = common.RunCommand("docker", append([]string{"wait"}, containerIDs...)...) - return err } func (d *DockerExecutor) listContainers() (map[string]ctr, error) { @@ -271,10 +278,6 @@ func (d *DockerExecutor) pollContainerIDs(ctx context.Context, containerNames [] log.Infof("ignoring container %q created at %v, too long before process started", containerName, c.createdAt) continue } - if c.status == "Created" && d.containers[containerName].status != "" { - log.Infof("ignoring created container %q that would %s -> %s", containerName, d.containers[containerName].status, c.status) - continue - } d.containers[containerName] = c log.Infof("mapped container name %q to container ID %q (created at %v, status %s)", containerName, c.containerID, c.createdAt, c.status) }