Skip to content

Commit

Permalink
Add configuration for create timeout (openshift#262)
Browse files Browse the repository at this point in the history
timeout is hard code and sometimes it's might be need to wait
for more than 5 min for instance create in some cloud.
  • Loading branch information
jichenjc authored and pierreprinetti committed Apr 22, 2024
1 parent e57b95d commit 18a0cd0
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 5 deletions.
6 changes: 6 additions & 0 deletions docs/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -280,3 +280,9 @@ Instead of tagging, you also have the option to add metadata to instances. This
securityGroups:
...
```
## Timeout settings
During some heavy workload cloud, the time for create and delete openstack instance might takes long time, by default it's 5 minute.
you can set:
`CLUSTER_API_OPENSTACK_INSTANCE_DELETE_TIMEOUT` for instance delete timeout value.
`CLUSTER_API_OPENSTACK_INSTANCE_CREATE_TIMEOUT` for instance create timeout value.
25 changes: 20 additions & 5 deletions pkg/cloud/openstack/machine/actuator.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ import (
"encoding/json"
"fmt"
"net"
"os"
"reflect"
"strconv"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -46,8 +48,8 @@ const (
DisableTemplatingKey = "disableTemplating"
PostprocessorKey = "postprocessor"

TimeoutInstanceCreate = 5 * time.Minute
TimeoutInstanceDelete = 5 * time.Minute
TimeoutInstanceCreate = 5
TimeoutInstanceDelete = 5
RetryIntervalInstanceStatus = 10 * time.Second

TokenTTL = 60 * time.Minute
Expand All @@ -69,6 +71,16 @@ func NewActuator(params openstack.ActuatorParams) (*OpenstackClient, error) {
}, nil
}

func getTimeout(name string, timeout int) time.Duration {
if v := os.Getenv(name); v != "" {
timeout, err := strconv.Atoi(v)
if err == nil {
return time.Duration(timeout)
}
}
return time.Duration(timeout)
}

func (oc *OpenstackClient) Create(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine) error {
kubeClient := oc.params.KubeClient

Expand Down Expand Up @@ -193,8 +205,9 @@ func (oc *OpenstackClient) Create(ctx context.Context, cluster *clusterv1.Cluste
return oc.handleMachineError(machine, apierrors.CreateMachine(
"error creating Openstack instance: %v", err))
}
// TODO: wait instance ready
err = util.PollImmediate(RetryIntervalInstanceStatus, TimeoutInstanceCreate, func() (bool, error) {
instanceCreateTimeout := getTimeout("CLUSTER_API_OPENSTACK_INSTANCE_CREATE_TIMEOUT", TimeoutInstanceCreate)
instanceCreateTimeout = instanceCreateTimeout * time.Minute
err = util.PollImmediate(RetryIntervalInstanceStatus, instanceCreateTimeout, func() (bool, error) {
instance, err := machineService.GetInstance(instance.ID)
if err != nil {
return false, nil
Expand Down Expand Up @@ -277,7 +290,9 @@ func (oc *OpenstackClient) Update(ctx context.Context, cluster *clusterv1.Cluste
if err != nil {
klog.Errorf("delete machine %s for update failed: %v", currentMachine.ObjectMeta.Name, err)
} else {
err = util.PollImmediate(RetryIntervalInstanceStatus, TimeoutInstanceDelete, func() (bool, error) {
instanceDeleteTimeout := getTimeout("CLUSTER_API_OPENSTACK_INSTANCE_DELETE_TIMEOUT", TimeoutInstanceDelete)
instanceDeleteTimeout = instanceDeleteTimeout * time.Minute
err = util.PollImmediate(RetryIntervalInstanceStatus, instanceDeleteTimeout, func() (bool, error) {
instance, err := oc.instanceExists(machine)
if err != nil {
return false, nil
Expand Down

0 comments on commit 18a0cd0

Please sign in to comment.