Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

pkg/operator: lay down ClusterOperator for MCO #386

Merged
merged 3 commits into from
Feb 9, 2019
Merged

pkg/operator: lay down ClusterOperator for MCO #386

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ee0e8e3
pkg/operator: lay down ClusterOperator for MCO
runcom Feb 6, 2019
bd2488f
pkg/operator: always report timeout errors
runcom Feb 8, 2019
34300cc
pkg/operator: wait longer for resources to be available to CVO
runcom Feb 8, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions install/0000_30_machine-config-operator_06_clusteroperator.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
apiVersion: config.openshift.io/v1
kind: ClusterOperator
metadata:
name: machine-config
spec: {}
70 changes: 45 additions & 25 deletions pkg/operator/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@ func (optr *Operator) syncAll(rconfig renderConfig) error {
// syncFuncs is the list of sync functions that are executed in order.
// any error marks sync as failure but continues to next syncFunc
var syncFuncs = []syncFunc{
{ "pools", optr.syncMachineConfigPools },
{ "mcc", optr.syncMachineConfigController },
{ "mcs", optr.syncMachineConfigServer },
{ "mcd", optr.syncMachineConfigDaemon },
{ "required-pools", optr.syncRequiredMachineConfigPools },
{"pools", optr.syncMachineConfigPools},
{"mcc", optr.syncMachineConfigController},
{"mcs", optr.syncMachineConfigServer},
{"mcd", optr.syncMachineConfigDaemon},
{"required-pools", optr.syncRequiredMachineConfigPools},
}

if err := optr.syncProgressingStatus(); err != nil {
Expand Down Expand Up @@ -329,7 +329,7 @@ func (optr *Operator) syncRequiredMachineConfigPools(config renderConfig) error
return err
}
var lastErr error
if err := wait.Poll(time.Second, 5*time.Minute, func() (bool, error) {
if err := wait.Poll(time.Second, 10*time.Minute, func() (bool, error) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel like these timeouts may still be too short. But we can revisit that later.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

well, the installer waits up to 30 mins anyway, we could bump up to that if needed I guess

pools, err := optr.mcpLister.List(sel)
if apierrors.IsNotFound(err) {
return false, err
Expand All @@ -353,10 +353,9 @@ func (optr *Operator) syncRequiredMachineConfigPools(config renderConfig) error
lastErr = fmt.Errorf("error pool %s is not ready. status: (total: %d, updated: %d, unavailable: %d)", p.Name, p.Status.MachineCount, p.Status.UpdatedMachineCount, p.Status.UnavailableMachineCount)
return false, nil
}

return true, nil
}); err != nil {
if err.Error() == wait.ErrWaitTimeout.Error() {
if err == wait.ErrWaitTimeout {
return fmt.Errorf("%v during syncRequiredMachineConfigPools: %v", err, lastErr)
}
return err
Expand All @@ -366,23 +365,24 @@ func (optr *Operator) syncRequiredMachineConfigPools(config renderConfig) error

const (
deploymentRolloutPollInterval = time.Second
deploymentRolloutTimeout = 5 * time.Minute
deploymentRolloutTimeout = 10 * time.Minute

daemonsetRolloutPollInterval = time.Second
daemonsetRolloutTimeout = 5 * time.Minute
daemonsetRolloutTimeout = 10 * time.Minute

customResourceReadyInterval = time.Second
customResourceReadyTimeout = 5 * time.Minute
customResourceReadyTimeout = 10 * time.Minute

controllerConfigCompletedInterval = time.Second
controllerConfigCompletedTimeout = time.Minute
controllerConfigCompletedTimeout = 5 * time.Minute
)

func (optr *Operator) waitForCustomResourceDefinition(resource *apiextv1beta1.CustomResourceDefinition) error {
return wait.Poll(customResourceReadyInterval, customResourceReadyTimeout, func() (bool, error) {
var lastErr error
if err := wait.Poll(customResourceReadyInterval, customResourceReadyTimeout, func() (bool, error) {
crd, err := optr.crdLister.Get(resource.Name)
if err != nil {
glog.Errorf("error getting CustomResourceDefinition %s: %v", resource.Name, err)
lastErr = fmt.Errorf("error getting CustomResourceDefinition %s: %v", resource.Name, err)
return false, nil
}

Expand All @@ -391,13 +391,20 @@ func (optr *Operator) waitForCustomResourceDefinition(resource *apiextv1beta1.Cu
return true, nil
}
}
glog.V(4).Infof("CustomResourceDefinition %s is not ready. conditions: %v", crd.Name, crd.Status.Conditions)
lastErr = fmt.Errorf("CustomResourceDefinition %s is not ready. conditions: %v", crd.Name, crd.Status.Conditions)
return false, nil
})
}); err != nil {
if err == wait.ErrWaitTimeout {
return fmt.Errorf("%v during syncCustomResourceDefinitions: %v", err, lastErr)
}
return err
}
return nil
}

func (optr *Operator) waitForDeploymentRollout(resource *appsv1.Deployment) error {
return wait.Poll(deploymentRolloutPollInterval, deploymentRolloutTimeout, func() (bool, error) {
var lastErr error
if err := wait.Poll(deploymentRolloutPollInterval, deploymentRolloutTimeout, func() (bool, error) {
d, err := optr.deployLister.Deployments(resource.Namespace).Get(resource.Name)
if apierrors.IsNotFound(err) {
// exit early to recreate the deployment.
Expand All @@ -406,7 +413,7 @@ func (optr *Operator) waitForDeploymentRollout(resource *appsv1.Deployment) erro
if err != nil {
// Do not return error here, as we could be updating the API Server itself, in which case we
// want to continue waiting.
glog.Errorf("error getting Deployment %s during rollout: %v", resource.Name, err)
lastErr = fmt.Errorf("error getting Deployment %s during rollout: %v", resource.Name, err)
return false, nil
}

Expand All @@ -417,13 +424,20 @@ func (optr *Operator) waitForDeploymentRollout(resource *appsv1.Deployment) erro
if d.Generation <= d.Status.ObservedGeneration && d.Status.UpdatedReplicas == d.Status.Replicas && d.Status.UnavailableReplicas == 0 {
return true, nil
}
glog.V(4).Infof("Deployment %s is not ready. status: (replicas: %d, updated: %d, ready: %d, unavailable: %d)", d.Name, d.Status.Replicas, d.Status.UpdatedReplicas, d.Status.ReadyReplicas, d.Status.UnavailableReplicas)
lastErr = fmt.Errorf("Deployment %s is not ready. status: (replicas: %d, updated: %d, ready: %d, unavailable: %d)", d.Name, d.Status.Replicas, d.Status.UpdatedReplicas, d.Status.ReadyReplicas, d.Status.UnavailableReplicas)
return false, nil
})
}); err != nil {
if err == wait.ErrWaitTimeout {
return fmt.Errorf("%v during waitForDeploymentRollout: %v", err, lastErr)
}
return err
}
return nil
}

func (optr *Operator) waitForDaemonsetRollout(resource *appsv1.DaemonSet) error {
return wait.Poll(daemonsetRolloutPollInterval, daemonsetRolloutTimeout, func() (bool, error) {
var lastErr error
if err := wait.Poll(daemonsetRolloutPollInterval, daemonsetRolloutTimeout, func() (bool, error) {
d, err := optr.daemonsetLister.DaemonSets(resource.Namespace).Get(resource.Name)
if apierrors.IsNotFound(err) {
// exit early to recreate the daemonset.
Expand All @@ -432,7 +446,7 @@ func (optr *Operator) waitForDaemonsetRollout(resource *appsv1.DaemonSet) error
if err != nil {
// Do not return error here, as we could be updating the API Server itself, in which case we
// want to continue waiting.
glog.Errorf("error getting Daemonset %s during rollout: %v", resource.Name, err)
lastErr = fmt.Errorf("error getting Daemonset %s during rollout: %v", resource.Name, err)
return false, nil
}

Expand All @@ -443,9 +457,15 @@ func (optr *Operator) waitForDaemonsetRollout(resource *appsv1.DaemonSet) error
if d.Generation <= d.Status.ObservedGeneration && d.Status.UpdatedNumberScheduled == d.Status.DesiredNumberScheduled && d.Status.NumberUnavailable == 0 {
return true, nil
}
glog.V(4).Infof("Daemonset %s is not ready. status: (desired: %d, updated: %d, ready: %d, unavailable: %d)", d.Name, d.Status.DesiredNumberScheduled, d.Status.UpdatedNumberScheduled, d.Status.NumberReady, d.Status.NumberAvailable)
lastErr = fmt.Errorf("Daemonset %s is not ready. status: (desired: %d, updated: %d, ready: %d, unavailable: %d)", d.Name, d.Status.DesiredNumberScheduled, d.Status.UpdatedNumberScheduled, d.Status.NumberReady, d.Status.NumberAvailable)
return false, nil
})
}); err != nil {
if err == wait.ErrWaitTimeout {
return fmt.Errorf("%v during waitForDaemonsetRollout: %v", err, lastErr)
}
return err
}
return nil
}

func (optr *Operator) waitForControllerConfigToBeCompleted(resource *mcfgv1.ControllerConfig) error {
Expand All @@ -457,7 +477,7 @@ func (optr *Operator) waitForControllerConfigToBeCompleted(resource *mcfgv1.Cont
}
return true, nil
}); err != nil {
if err.Error() == wait.ErrWaitTimeout.Error() {
if err == wait.ErrWaitTimeout {
return fmt.Errorf("%v during waitForControllerConfigToBeCompleted: %v", err, lastErr)
}
return err
Expand Down