enlarge timeout when pouch start

ctrd/container.go

@@ -318,27 +318,36 @@ func (c *Client) recoverContainer(ctx context.Context, id string, io *containeri
 	}
 
 	var (
-		timeout = 5 * time.Second
+		timeout = 3 * time.Second
 		ch      = make(chan error, 1)
 		task    containerd.Task
 	)
 
 	// for normal shim, this operation should be end less than 1 second,
 	// we give 5 second timeout to believe the shim get locked internal,
 	// return error since we do not want a hang shim affect daemon start
-	pctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	go func() {
-		task, err = lc.Task(pctx, func(fset *cio.FIFOSet) (cio.IO, error) {
-			return c.attachIO(fset, io.InitContainerIO)
-		})
-		ch <- err
-	}()
+	// XXX: when system load is high, make connect to shim fail on retry 3 times
+	for i := 0; i < 3; i++ {
+		pctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		go func() {
+			task, err = lc.Task(pctx, func(fset *cio.FIFOSet) (cio.IO, error) {
+				return c.attachIO(fset, io.InitContainerIO)
+			})
+			ch <- err
+		}()
+
+		select {
+		case <-time.After(timeout):
+			if i < 2 {
+				logrus.WithField("container", id).Warn("timeout connect to shim, retry")
+				continue
+			}
+			return errors.Wrap(errtypes.ErrTimeout, "failed to connect to shim")
+		case err = <-ch:
+		}
 
-	select {
-	case <-time.After(timeout):
-		return errors.Wrap(errtypes.ErrTimeout, "failed to connect to shim")
-	case err = <-ch:
+		break
 	}
 
 	if err != nil {

ctrd/supervisord/daemon.go

@@ -138,13 +138,14 @@ func (d *Daemon) Stop() error {
 func (d *Daemon) healthPostCheck() error {
 	var (
 		failureCount  = 0
-		maxRetryCount = 3
+		maxRetryCount = 10
 		client        *containerd.Client
 		err           error
 	)
 
+	// shim v2 has a connect timeout, enlarge the postcheck timeout
 	for ; failureCount < maxRetryCount; failureCount++ {
-		time.Sleep(time.Duration(failureCount*500) * time.Millisecond)
+		time.Sleep(time.Duration(failureCount) * time.Second)
 
 		if client == nil {
 			client, err = containerd.New(d.Address())