Retry containerd task api request when task is in unknown state

In some cases, it appears that containerd task api can return a zero pid
when the task is not ready yet. In these cases, the process task state
is reported to be in unknown state. In this case, we should retry the
containerd task api request as the task can take some time to move away
from unknown state and be initialized.

Signed-off-by: David Porter <porterdavid@google.com>

container/containerd/client.go

@@ -16,6 +16,7 @@ package containerd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"sync"
@@ -24,6 +25,7 @@ import (
 	containersapi "github.com/containerd/containerd/api/services/containers/v1"
 	tasksapi "github.com/containerd/containerd/api/services/tasks/v1"
 	versionapi "github.com/containerd/containerd/api/services/version/v1"
+	tasktypes "github.com/containerd/containerd/api/types/task"
 	ptypes "github.com/gogo/protobuf/types"
 	"github.com/google/cadvisor/container/containerd/containers"
 	"github.com/google/cadvisor/container/containerd/errdefs"
@@ -44,6 +46,10 @@ type ContainerdClient interface {
 	Version(ctx context.Context) (string, error)
 }
 
+var (
+	ErrTaskIsInUnknownState = errors.New("containerd task is in unknown state") // used when containerd task api is in nUnknownState
+)
+
 var once sync.Once
 var ctrdClient ContainerdClient = nil
 
@@ -114,6 +120,9 @@ func (c *client) TaskPid(ctx context.Context, id string) (uint32, error) {
 	if err != nil {
 		return 0, errdefs.FromGRPC(err)
 	}
+	if response.Process.Status == tasktypes.StatusUnknown {
+		return 0, ErrTaskIsInUnknownState
+	}
 	return response.Process.Pid, nil
 }
 

container/containerd/handler.go

@@ -17,6 +17,7 @@ package containerd
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -101,10 +102,13 @@ func newContainerdContainerHandler(
 		if err == nil {
 			break
 		}
-		retry--
-		if !errdefs.IsNotFound(err) || retry == 0 {
+
+		isRetriableError := errdefs.IsNotFound(err) || errors.Is(err, ErrTaskIsInUnknownState)
+		if !isRetriableError || retry == 0 {
 			return nil, err
 		}
+
+		retry--
 		time.Sleep(backoff)
 		backoff *= 2
 	}