Functional tests

src/ApiService/ApiService/Functions/Proxy.cs

@@ -92,7 +92,7 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
                 "ProxyCreate");
         }
 
-        var scaleset = await _context.ScalesetOperations.GetById(request.OkV.MachineId);
+        var scaleset = await _context.ScalesetOperations.GetById(request.OkV.ScalesetId);
         if (!scaleset.IsOk) {
             return await _context.RequestHandling.NotOk(req, scaleset.ErrorV, "debug_proxy create");
         }
@@ -112,7 +112,7 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
         var proxy = await _context.ProxyOperations.GetOrCreate(scaleset.OkV.Region);
         if (proxy != null) {
             var updated = forwardResult.OkV with { ProxyId = proxy.ProxyId };
-            await _context.ProxyForwardOperations.Update(updated);
+            await _context.ProxyForwardOperations.Replace(updated);
             await _context.ProxyOperations.SaveProxyConfig(proxy);
         }
 

src/ApiService/ApiService/onefuzzlib/IpOperations.cs

@@ -37,7 +37,7 @@ public class IpOperations : IIpOperations {
     public IpOperations(ILogTracer log, IOnefuzzContext context) {
         _logTracer = log;
         _context = context;
-        _networkInterfaceQuery = new NetworkInterfaceQuery(context);
+        _networkInterfaceQuery = new NetworkInterfaceQuery(context, log);
     }
 
     public async Async.Task<NetworkInterfaceResource?> GetPublicNic(string resourceGroup, string name) {
@@ -219,13 +219,15 @@ record ValueList<T>(List<T> value);
 
         private readonly IOnefuzzContext _context;
 
-        public NetworkInterfaceQuery(IOnefuzzContext context) {
+        private readonly ILogTracer _logTracer;
+
+        public NetworkInterfaceQuery(IOnefuzzContext context, ILogTracer logTracer) {
             _context = context;
+            _logTracer = logTracer;
         }
 
 
         public async Task<List<string>> ListInstancePrivateIps(Guid scalesetId, string instanceId) {
-
             var token = _context.Creds.GetIdentity().GetToken(
                 new TokenRequestContext(
                     new[] { $"https://management.azure.com" }));
@@ -234,13 +236,15 @@ public async Task<List<string>> ListInstancePrivateIps(Guid scalesetId, string i
             client.DefaultRequestHeaders.Add("Authorization", "Bearer " + token.Token);
             var baseUrl = new Uri($"https://management.azure.com/");
             // https://docs.microsoft.com/en-us/rest/api/virtualnetwork/network-interface-in-vm-ss/get-virtual-machine-scale-set-network-interface?tabs=HTTP
-            var requestURl = baseUrl + $"subscriptions/{_context.Creds.GetSubscription()}/resourceGroups/{_context.Creds.GetBaseResourceGroup()}/providers/Microsoft.Compute/virtualMachineScaleSets/{scalesetId}/virtualMachines/{instanceId}/networkInterfaces?api-version=2021-08-01";
+            var requestURl = baseUrl + $"subscriptions/{_context.Creds.GetSubscription()}/resourceGroups/{_context.Creds.GetBaseResourceGroup()}/providers/Microsoft.Compute/virtualMachineScaleSets/{scalesetId}/virtualMachines/{instanceId}/networkInterfaces?api-version=2022-08-01";
             var response = await client.GetAsync(requestURl);
             if (response.IsSuccessStatusCode) {
                 var responseStream = await response.Content.ReadAsStreamAsync();
                 var nics = await JsonSerializer.DeserializeAsync<ValueList<NetworkInterface>>(responseStream);
                 if (nics != null)
                     return nics.value.SelectMany(x => x.properties.ipConfigurations.Select(i => i.properties.privateIPAddress)).WhereNotNull().ToList();
+            } else {
+                _logTracer.Error($"failed to get ListInstancePrivateIps due to {await response.Content.ReadAsStringAsync()}");
             }
             return new List<string>();
         }

src/ApiService/ApiService/onefuzzlib/ProxyForwardOperations.cs

@@ -76,7 +76,7 @@ public async Task<OneFuzzResult<ProxyForward>> UpdateOrCreate(string region, Gui
                 ProxyId: null
             );
 
-            var result = await Insert(entry);
+            var result = await Replace(entry);
             if (!result.IsOk) {
                 _logTracer.Info($"port is already used {entry}");
             }

src/ApiService/FunctionalTests/1f-api/ApiBase.cs

@@ -5,7 +5,21 @@
 
 namespace FunctionalTests;
 
-abstract class ApiBase<T> where T : new() {
+interface IFromJsonElement<T> {
+    T Convert(JsonElement e);
+}
+
+class BooleanResult : IFromJsonElement<BooleanResult> {
+    JsonElement _e;
+    public BooleanResult() { }
+    public BooleanResult(JsonElement e) => _e = e;
+
+    public bool Result => _e.GetProperty("result").GetBoolean();
+
+    public BooleanResult Convert(JsonElement e) => new BooleanResult(e);
+}
+
+abstract class ApiBase {
 
     Uri _endpoint;
     Microsoft.OneFuzz.Service.Request _request;
@@ -17,8 +31,6 @@ public ApiBase(Uri endpoint, string relativeUri, Microsoft.OneFuzz.Service.Reque
         _output = output;
     }
 
-    public abstract T Convert(JsonElement e);
-
     public async Task<JsonElement> Get(JsonObject root) {
         var body = root.ToJsonString();
         var r = await _request.Get(_endpoint, body);
@@ -44,32 +56,32 @@ public async Task<JsonElement> Delete(JsonObject root) {
         return (await JsonDocument.ParseAsync(r.Content.ReadAsStream())).RootElement;
     }
 
-    public Result<IEnumerable<T>, Error> IEnumerableResult(JsonElement res) {
+    public static Result<IEnumerable<T>, Error> IEnumerableResult<T>(JsonElement res) where T : IFromJsonElement<T>, new() {
         if (Error.IsError(res)) {
             return Result<IEnumerable<T>, Error>.Error(new Error(res));
         } else {
             if (res.ValueKind == JsonValueKind.Array)
-                return Result<IEnumerable<T>, Error>.Ok(res.EnumerateArray().Select(e => Convert(e)));
+                return Result<IEnumerable<T>, Error>.Ok(res.EnumerateArray().Select(e => (new T()).Convert(e)));
             else {
-                var r = Result(res);
+                var r = Result<T>(res);
                 if (r.IsOk)
                     return Result<IEnumerable<T>, Error>.Ok(new[] { r.OkV });
                 else
                     return Result<IEnumerable<T>, Error>.Error(r.ErrorV);
             }
         }
     }
-    public Result<T, Error> Result(JsonElement res) {
+    public static Result<T, Error> Result<T>(JsonElement res) where T : IFromJsonElement<T>, new() {
         if (Error.IsError(res)) {
             return Result<T, Error>.Error(new Error(res));
         } else {
             Assert.True(res.ValueKind != JsonValueKind.Array);
-            return Result<T, Error>.Ok(Convert(res));
+            return Result<T, Error>.Ok((new T()).Convert(res));
         }
     }
 
-    public static bool DeleteResult(JsonElement res) {
-        return res.GetProperty("result").GetBoolean();
+    public static T Return<T>(JsonElement res) where T : IFromJsonElement<T>, new() {
+        return (new T()).Convert(res);
     }
 
 }

src/ApiService/FunctionalTests/1f-api/Error.cs

@@ -3,7 +3,7 @@
 
 namespace FunctionalTests;
 
-class Error : IComparable<Error> {
+class Error : IComparable<Error>, IFromJsonElement<Error> {
     JsonElement _e;
 
     public Error(JsonElement e) {
@@ -15,6 +15,8 @@ public Error(JsonElement e) {
 
     public IEnumerable<string> Errors => _e.GetProperty("errors").EnumerateArray().Select(e => e.GetString()!);
 
+    public Error Convert(JsonElement e) => new Error(e);
+
     public static bool IsError(JsonElement res) {
         return res.ValueKind == JsonValueKind.Object && res.TryGetProperty("code", out _) && res.TryGetProperty("errors", out _);
     }
@@ -46,4 +48,6 @@ public override string ToString() {
     public bool UnableToFindPoolError => Code == 450 && Errors.First() == "unable to find pool";
 
     public bool UnableToFindScalesetError => Code == 450 && Errors.First() == "unable to find scaleset";
+
+    public bool UnableToFindNode => Code == 467 && Errors.First() == "unable to find node ";
 }

src/ApiService/FunctionalTests/1f-api/Node.cs

@@ -0,0 +1,99 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace FunctionalTests;
+
+class Node : IFromJsonElement<Node> {
+    JsonElement _e;
+
+    public Node() { }
+
+    public Node(JsonElement e) => _e = e;
+
+    public string PoolName => _e.GetProperty("pool_name").GetString()!;
+
+    public Guid MachineId => _e.GetProperty("machine_id").GetGuid();
+
+    public Guid? PoolId => _e.GetProperty("pool_id").ValueKind == JsonValueKind.Null ? null : _e.GetProperty("pool_id").GetGuid();
+
+    public string Version => _e.GetProperty("version").GetString()!;
+
+    public DateTimeOffset? HeartBeat => _e.GetProperty("heart_beat").ValueKind == JsonValueKind.Null ? null : _e.GetProperty("heart_beat").GetDateTimeOffset();
+
+    public DateTimeOffset? InitializedAt => _e.GetProperty("initialized_at").ValueKind == JsonValueKind.Null ? null : _e.GetProperty("initialized_at").GetDateTimeOffset();
+
+    public string State => _e.GetProperty("state").GetString()!;
+
+    public Guid? ScalesetId => _e.GetProperty("scaleset_id").ValueKind == JsonValueKind.Null ? null : _e.GetProperty("scaleset_id").GetGuid();
+    public bool ReimageRequested => _e.GetProperty("reimage_requested").GetBoolean();
+    public bool DeleteRequested => _e.GetProperty("delete_requested").GetBoolean();
+    public bool DebugKeepNode => _e.GetProperty("debug_keep_node").GetBoolean();
+
+    public Node Convert(JsonElement e) => new Node(e);
+}
+
+
+class NodeApi : ApiBase {
+
+    public NodeApi(Uri endpoint, Microsoft.OneFuzz.Service.Request request, ITestOutputHelper output) :
+        base(endpoint, "/api/Node", request, output) {
+    }
+
+    public async Task<JsonElement> Update(Guid machineId, bool? debugKeepNode = null) {
+        var j = new JsonObject();
+        if (debugKeepNode is not null)
+            j.Add("debug_keep_node", JsonValue.Create(debugKeepNode));
+
+        j.Add("machine_id", JsonValue.Create(machineId));
+        return await Post(j);
+    }
+    public async Task<Result<IEnumerable<Node>, Error>> Get(Guid? machineId = null, List<string>? state = null, Guid? scalesetId = null, string? poolName = null) {
+        var j = new JsonObject();
+        if (machineId is not null)
+            j.Add("machine_id", JsonValue.Create(machineId));
+        if (state is not null) {
+            var states = new JsonArray(state.Select(s => JsonValue.Create(s)).ToArray());
+            j.Add("state", JsonValue.Create(states));
+        }
+        if (scalesetId is not null)
+            j.Add("scaleset_id", JsonValue.Create(scalesetId));
+
+        if (poolName is not null)
+            j.Add("pool_name", JsonValue.Create(poolName));
+
+        return IEnumerableResult<Node>(await Get(j));
+    }
+
+    public async Task<JsonElement> Patch(Guid machineId) {
+        var j = new JsonObject();
+        j.Add("machine_id", JsonValue.Create(machineId));
+        return await Patch(j);
+    }
+
+    public async Task<BooleanResult> Delete(Guid machineId) {
+        var j = new JsonObject();
+        j.Add("machine_id", JsonValue.Create(machineId));
+        return Return<BooleanResult>(await Delete(j));
+    }
+
+    public async Task<Node> WaitWhile(Guid id, Func<Node, bool> wait) {
+        var currentState = "";
+        Node node;
+        do {
+            await Task.Delay(TimeSpan.FromSeconds(10.0));
+            var sc = await Get(machineId: id);
+            Assert.True(sc.IsOk, $"failed to get node with id: {id} due to {sc.ErrorV}");
+            node = sc.OkV!.First();
+
+            if (currentState != node.State) {
+                _output.WriteLine($"Node is in {node.State}");
+                currentState = node.State;
+            }
+        } while (wait(node));
+
+        return node;
+    }
+
+}

src/ApiService/FunctionalTests/1f-api/Pool.cs

@@ -5,7 +5,7 @@
 
 namespace FunctionalTests;
 
-class Pool {
+class Pool : IFromJsonElement<Pool> {
 
     JsonElement _e;
 
@@ -15,22 +15,23 @@ public Pool() { }
     public string Name => _e.GetProperty("name").GetString()!;
     public string PoolId => _e.GetProperty("pool_id").GetString()!;
     public string Os => _e.GetProperty("os").GetString()!;
+
+    public Pool Convert(JsonElement e) => new Pool(e);
 }
 
-class PoolApi : ApiBase<Pool> {
+class PoolApi : ApiBase {
 
     public static string TestPoolPrefix = "FT-DELETE-";
 
     public PoolApi(Uri endpoint, Microsoft.OneFuzz.Service.Request request, ITestOutputHelper output) :
         base(endpoint, "/api/Pool", request, output) {
     }
-    public override Pool Convert(JsonElement e) { return new Pool(e); }
 
-    public async Task<bool> Delete(string name, bool now = true) {
+    public async Task<BooleanResult> Delete(string name, bool now = true) {
         var root = new JsonObject();
         root.Add("name", JsonValue.Create(name));
         root.Add("now", JsonValue.Create(now));
-        return DeleteResult(await Delete(root));
+        return Return<BooleanResult>(await Delete(root));
     }
 
     public async Task<Result<IEnumerable<Pool>, Error>> Get(string? poolName = null, string? poolId = null, string? state = null) {
@@ -40,7 +41,7 @@ public async Task<Result<IEnumerable<Pool>, Error>> Get(string? poolName = null,
         root.Add("state", state);
 
         var res = await Get(root);
-        return IEnumerableResult(res);
+        return IEnumerableResult<Pool>(res);
     }
 
     public async Task DeleteAll() {
@@ -53,7 +54,7 @@ public async Task DeleteAll() {
             if (pool.Name.StartsWith(TestPoolPrefix)) {
                 _output.WriteLine($"Deleting {pool.Name}");
                 var deleted = await Delete(pool.Name);
-                Assert.True(deleted);
+                Assert.True(deleted.Result);
             }
         }
     }
@@ -64,6 +65,6 @@ public async Task<Result<Pool, Error>> Create(string poolName, string os, string
         rootPoolCreate.Add("os", os);
         rootPoolCreate.Add("arch", arch);
         rootPoolCreate.Add("managed", true);
-        return Result(await Post(rootPoolCreate));
+        return Result<Pool>(await Post(rootPoolCreate));
     }
 }