Skip to content
This repository has been archived by the owner on Nov 1, 2023. It is now read-only.

Improve error reporting from scale-in protection modification #3184

Merged
merged 2 commits into from
Jun 9, 2023
Merged

Improve error reporting from scale-in protection modification #3184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dec09bc
Improve error reporting from scale-in protection modification
tevoinea Jun 9, 2023
25ccdfd
Merge branch 'main' into tevoinea/MoreGranularScaleInProtectionErrors
tevoinea Jun 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/ApiService/ApiService/OneFuzzTypes/Enums.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ public enum ErrorCode {
UNABLE_TO_SEND = 483,
NODE_DELETED = 484,
TASK_CANCELLED = 485,
SCALE_IN_PROTECTION_UPDATE_ALREADY_IN_PROGRESS = 486,
SCALE_IN_PROTECTION_INSTANCE_NO_LONGER_EXISTS = 487,
SCALE_IN_PROTECTION_REACHED_MODEL_LIMIT = 488,
SCALE_IN_PROTECTION_UNEXPECTED_ERROR = 489,
// NB: if you update this enum, also update enums.py
}

Expand Down
19 changes: 17 additions & 2 deletions src/ApiService/ApiService/onefuzzlib/NodeOperations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,14 @@ await TryGetNodeInfo(node) is NodeInfo nodeInfo) {

var r = await _context.VmssOperations.UpdateScaleInProtection(nodeInfo.Scaleset, instanceId, protectFromScaleIn: true);
if (!r.IsOk) {
_logTracer.LogOneFuzzError(r.ErrorV);
switch (r.ErrorV.Code) {
case ErrorCode.SCALE_IN_PROTECTION_UPDATE_ALREADY_IN_PROGRESS:
tevoinea marked this conversation as resolved.
Show resolved Hide resolved
_logTracer.LogWarning("Transiently failed to modify scale-in protection: {}", r.ErrorV);
break;
default:
_logTracer.LogOneFuzzError(r.ErrorV);
break;
}
_logTracer.LogMetric("FailedAcquiringScaleInProtection", 1);
return r.ErrorV;
}
Expand Down Expand Up @@ -148,7 +155,15 @@ await TryGetNodeInfo(node) is NodeInfo nodeInfo) {

var r = await _context.VmssOperations.UpdateScaleInProtection(nodeInfo.Scaleset, instanceId, protectFromScaleIn: false);
if (!r.IsOk) {
_logTracer.LogOneFuzzError(r.ErrorV);
switch (r.ErrorV.Code) {
case ErrorCode.SCALE_IN_PROTECTION_INSTANCE_NO_LONGER_EXISTS:
case ErrorCode.SCALE_IN_PROTECTION_UPDATE_ALREADY_IN_PROGRESS:
_logTracer.LogWarning("Transiently failed to modify scale-in protection: {}", r.ErrorV);
break;
default:
_logTracer.LogOneFuzzError(r.ErrorV);
break;
}
_logTracer.LogMetric("FailedReleasingScaleInProtection", 1);
return r;
}
Expand Down
8 changes: 4 additions & 4 deletions src/ApiService/ApiService/onefuzzlib/VmssOperations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,14 +253,14 @@ public async Async.Task<OneFuzzResultVoid> UpdateScaleInProtection(Scaleset scal
_ = await vmCollection.CreateOrUpdateAsync(WaitUntil.Started, instanceId, data);
return OneFuzzResultVoid.Ok;
} catch (RequestFailedException ex) when (ex.Status == 409 && ex.Message.StartsWith("The request failed due to conflict with a concurrent request")) {
return OneFuzzResultVoid.Error(ErrorCode.UNABLE_TO_UPDATE, $"protection policy update is already in progress: {instanceId} in vmss {scaleset.ScalesetId}");
return OneFuzzResultVoid.Error(ErrorCode.SCALE_IN_PROTECTION_UPDATE_ALREADY_IN_PROGRESS, $"protection policy update is already in progress: {instanceId} in vmss {scaleset.ScalesetId}");
} catch (RequestFailedException ex) when (ex.Status == 400 && ex.Message.Contains("The provided instanceId") && ex.Message.Contains("not an active Virtual Machine Scale Set VM instanceId.")) {
return OneFuzzResultVoid.Error(ErrorCode.UNABLE_TO_UPDATE, $"The node with instanceId {instanceId} no longer exists in scaleset {scaleset.ScalesetId}");
return OneFuzzResultVoid.Error(ErrorCode.SCALE_IN_PROTECTION_INSTANCE_NO_LONGER_EXISTS, $"The node with instanceId {instanceId} no longer exists in scaleset {scaleset.ScalesetId}");
} catch (RequestFailedException ex) when (ex.Status == 400 && ex.Message.Contains("reached its limit") && ex.Message.Contains("Upgrade the VMs to the latest model")) {
return OneFuzzResultVoid.Error(ErrorCode.UNABLE_TO_UPDATE, $"VMSS has reached model limit. Could not update scaling protection for scaleset {scaleset.ScalesetId}.");
return OneFuzzResultVoid.Error(ErrorCode.SCALE_IN_PROTECTION_REACHED_MODEL_LIMIT, $"VMSS has reached model limit. Could not update scaling protection for scaleset {scaleset.ScalesetId}.");
} catch (Exception ex) {
_log.LogError(ex, "unable to set protection policy on: {InstanceId} in vmss {ScalesetId}", instanceId, scaleset.ScalesetId);
return OneFuzzResultVoid.Error(ErrorCode.UNABLE_TO_UPDATE, $"unable to set protection policy on: {instanceId} in vmss {scaleset.ScalesetId}");
return OneFuzzResultVoid.Error(ErrorCode.SCALE_IN_PROTECTION_UNEXPECTED_ERROR, $"unable to set protection policy on: {instanceId} in vmss {scaleset.ScalesetId}");
}

}
Expand Down
4 changes: 4 additions & 0 deletions src/pytypes/onefuzztypes/enums.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,10 @@ class ErrorCode(Enum):
UNABLE_TO_SEND = 483
NODE_DELETED = 484
TASK_CANCELLED = 485
SCALE_IN_PROTECTION_UPDATE_ALREADY_IN_PROGRESS = 486
SCALE_IN_PROTECTION_INSTANCE_NO_LONGER_EXISTS = 487
SCALE_IN_PROTECTION_REACHED_MODEL_LIMIT = 488
SCALE_IN_PROTECTION_UNEXPECTED_ERROR = 489
# NB: if you update this enum, also update Enums.cs


Expand Down