Skip to content

Commit

Permalink
Report extension errors (microsoft#2846)
Browse files Browse the repository at this point in the history
Old failure message:
```
failed to launch extension
```

New failure message:

```
failed to launch extension(s): Errors for extension 'CustomScriptExtension':
:Error: ProvisioningState/failed/3 (Provisioning failed) - Failed to download all specified files. Exiting. Error Message: The remote server returned an error: (400) Bad Request.
```
  • Loading branch information
Porges authored and chkeita committed Feb 16, 2023
1 parent 2b3e15e commit 2a191e1
Showing 1 changed file with 41 additions and 18 deletions.
59 changes: 41 additions & 18 deletions src/ApiService/ApiService/onefuzzlib/VmOperations.cs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
using System.Threading.Tasks;
using System.Text;
using System.Threading.Tasks;
using Azure;
using Azure.Core;
using Azure.ResourceManager.Compute;
using Azure.ResourceManager.Compute.Models;
using Newtonsoft.Json;

namespace Microsoft.OneFuzz.Service;

Expand Down Expand Up @@ -153,7 +154,7 @@ public async System.Threading.Tasks.Task DeleteVm(string name) {


public async Task<OneFuzzResult<bool>> AddExtensions(Vm vm, Dictionary<string, VirtualMachineExtensionData> extensions) {
var status = new List<string>();
var statuses = new List<(string extName, string state)>();
var toCreate = new List<KeyValuePair<string, VirtualMachineExtensionData>>();
foreach (var extensionConfig in extensions) {
var extensionName = extensionConfig.Key;
Expand All @@ -164,7 +165,7 @@ public async Task<OneFuzzResult<bool>> AddExtensions(Vm vm, Dictionary<string, V
_logTracer.Info(
$"vm extension state: {vm.Name:Tag:VmName} - {extensionName:Tag:ExtensionName} - {extension.ProvisioningState:Tag:ExtensionProvisioningState}"
);
status.Add(extension.ProvisioningState);
statuses.Add((extensionName, extension.ProvisioningState));
} else {
toCreate.Add(extensionConfig);
}
Expand All @@ -175,19 +176,15 @@ public async Task<OneFuzzResult<bool>> AddExtensions(Vm vm, Dictionary<string, V
await CreateExtension(vm.Name, config.Key, config.Value);
}
} else {
if (status.All(s => string.Equals(s, "Succeeded", StringComparison.Ordinal))) {
return OneFuzzResult<bool>.Ok(true);
} else if (status.Any(s => string.Equals(s, "Failed", StringComparison.Ordinal))) {
return OneFuzzResult<bool>.Error(
ErrorCode.VM_CREATE_FAILED,
"failed to launch extension"
);
} else if (!(status.Contains("Creating") || status.Contains("Updating"))) {
_logTracer.Error($"vm agent - unknown state {vm.Name:Tag:VmName}: {JsonConvert.SerializeObject(status):Tag:Status}");
if (statuses.All(s => s.state == "Succeeded")) {
return OneFuzzResult.Ok(true);
} else if (statuses.Any(s => s.state == "Failed")) {
var errors = await GetExtensionErrors(vm.Name, statuses.Where(s => s.state == "Failed").Select(s => s.extName));
return OneFuzzResult<bool>.Error(ErrorCode.VM_CREATE_FAILED, "failed to launch extension(s): " + errors);
}
}

return OneFuzzResult<bool>.Ok(false);
return OneFuzzResult.Ok(false);
}

public async Task<OneFuzzResultVoid> Create(Vm vm) {
Expand Down Expand Up @@ -223,16 +220,21 @@ public async Task<OneFuzzResultVoid> Create(Vm vm) {
}
}

private ResourceIdentifier GetVirtualMachineIdentifier(string vmName)
=> VirtualMachineResource.CreateResourceIdentifier(
_context.Creds.GetSubscription(),
_context.Creds.GetBaseResourceGroup(),
vmName);

public async Async.Task CreateExtension(string vmName, string extensionName, VirtualMachineExtensionData extension) {
_logTracer.Info($"creating extension: {_context.Creds.GetBaseResourceGroup():Tag:ResourceGroup} - {vmName:Tag:VmName} - {extensionName:Tag:ExtensionName}");
var vm = await _context.Creds.GetResourceGroupResource().GetVirtualMachineAsync(vmName);

var vm = _context.Creds.ArmClient.GetVirtualMachineResource(GetVirtualMachineIdentifier(vmName));
try {
_ = await vm.Value.GetVirtualMachineExtensions().CreateOrUpdateAsync(
_ = await vm.GetVirtualMachineExtensions().CreateOrUpdateAsync(
WaitUntil.Started,
extensionName,
extension
);
extension);
} catch (RequestFailedException ex) when
(ex.Status == 409 &&
(ex.Message.Contains("VM is marked for deletion") || ex.Message.Contains("The request failed due to conflict with a concurrent request."))) {
Expand All @@ -241,6 +243,27 @@ public async Async.Task CreateExtension(string vmName, string extensionName, Vir
return;
}

public async Async.Task<string> GetExtensionErrors(string vmName, IEnumerable<string> extensionNames) {
var vmResource = _context.Creds.ArmClient.GetVirtualMachineResource(GetVirtualMachineIdentifier(vmName));
var vmData = await vmResource.GetAsync(InstanceViewTypes.InstanceView);

var result = new StringBuilder();
foreach (var extensionName in extensionNames) {
result.Append($"Errors for extension '{extensionName}':");
var extensionData = vmData.Value.Data.InstanceView.Extensions.FirstOrDefault(ext => ext.Name == extensionName);
if (extensionData is null) {
result.Append($" (cannot get errors - extension was not found on target VM)\n");
} else {
result.Append('\n');
foreach (var status in extensionData.Statuses) {
result.Append($"{status.Time}:{status.Level}: {status.Code} ({status.DisplayStatus}) - {status.Message}\n");
}
}
}

return result.ToString();
}

async Task<OneFuzzResultVoid> CreateVm(
string name,
Region location,
Expand Down

0 comments on commit 2a191e1

Please sign in to comment.