migrate timer_proxy part 2

src/ApiService/ApiService/ApiService.csproj

@@ -8,6 +8,7 @@
     <WarningLevel>5</WarningLevel>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="Faithlife.Utility" Version="0.12.2" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Storage" Version="5.0.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.EventGrid" Version="2.1.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.0.13" />

src/ApiService/ApiService/OneFuzzTypes/Events.cs

@@ -1,4 +1,4 @@
-using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
+using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using PoolName = System.String;

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -95,7 +95,7 @@ public record ProxyHeartbeat
     DateTimeOffset TimeStamp
 );
 
-public partial record Node
+public record Node
 (
     DateTimeOffset? InitializedAt,
     [PartitionKey] PoolName PoolName,
@@ -111,27 +111,40 @@ bool DebugKeepNode
 ) : StatefulEntityBase<NodeState>(State);
 
 
-public partial record ProxyForward
+public record Forward
 (
-    [PartitionKey] Region Region,
-    [RowKey] int DstPort,
     int SrcPort,
+    int DstPort,
     string DstIp
+);
+
+
+public record ProxyForward
+(
+    [PartitionKey] Region Region,
+    int Port,
+    Guid ScalesetId,
+    Guid MachineId,
+    Guid? ProxyId,
+    [RowKey] int DstPort,
+    string DstIp,
+    DateTimeOffset EndTime
 ) : EntityBase();
 
-public partial record ProxyConfig
+public record ProxyConfig
 (
     Uri Url,
-    string Notification,
+    Uri Notification,
     Region Region,
     Guid? ProxyId,
-    List<ProxyForward> Forwards,
+    List<Forward> Forwards,
     string InstanceTelemetryKey,
-    string MicrosoftTelemetryKey
+    string MicrosoftTelemetryKey,
+    Guid InstanceId
 
 );
 
-public partial record Proxy
+public record Proxy
 (
     [PartitionKey] Region Region,
     [RowKey] Guid ProxyId,

src/ApiService/ApiService/Program.cs

@@ -1,4 +1,4 @@
-// to avoid collision with Task in model.cs
+// to avoid collision with Task in model.cs
 global using Async = System.Threading.Tasks;
 
 global using System;

src/ApiService/ApiService/ServiceConfiguration.cs

@@ -36,6 +36,8 @@ public interface IServiceConfig
 
     public string? OneFuzzResourceGroup { get; }
     public string? OneFuzzTelemetry { get; }
+
+    public string OnefuzzVersion { get; }
 }
 
 public class ServiceConfiguration : IServiceConfig
@@ -79,4 +81,5 @@ public ServiceConfiguration()
     public string? OneFuzzOwner { get => Environment.GetEnvironmentVariable("ONEFUZZ_OWNER"); }
     public string? OneFuzzResourceGroup { get => Environment.GetEnvironmentVariable("ONEFUZZ_RESOURCE_GROUP"); }
     public string? OneFuzzTelemetry { get => Environment.GetEnvironmentVariable("ONEFUZZ_TELEMETRY"); }
+    public string OnefuzzVersion { get => Environment.GetEnvironmentVariable("ONEFUZZ_VERSION") ?? "0.0.0"; }
 }

src/ApiService/ApiService/onefuzzlib/Auth.cs

@@ -0,0 +1,16 @@
+namespace Microsoft.OneFuzz.Service;
+
+using System.Security.Cryptography;
+
+public class Auth
+{
+    public static Authentication BuildAuth()
+    {
+        var rsa = RSA.Create(2048);
+        string header = "-----BEGIN RSA PRIVATE KEY-----";
+        string footer = "-----END RSA PRIVATE KEY-----";
+        var privateKey = $"{header}\n{Convert.ToBase64String(rsa.ExportRSAPrivateKey())}\n{footer}";
+        var publiceKey = $"{header}\n{Convert.ToBase64String(rsa.ExportRSAPublicKey())}\n{footer}";
+        return new Authentication(Guid.NewGuid().ToString(), publiceKey, privateKey);
+    }
+}

src/ApiService/ApiService/onefuzzlib/Containers.cs

@@ -3,17 +3,20 @@
 using Azure.Storage.Blobs;
 using Azure.Storage;
 using Azure;
+using Azure.Storage.Sas;
 
 namespace Microsoft.OneFuzz.Service;
 
+
 public interface IContainers
 {
-    public Task<IEnumerable<byte>?> GetBlob(Container container, string name, StorageType storageType);
+    public Task<BinaryData?> GetBlob(Container container, string name, StorageType storageType);
 
     public Async.Task<BlobContainerClient?> FindContainer(Container container, StorageType storageType);
 
-    public Uri GetFileSasUrl(Container container, string name, StorageType storageType, bool read = false, bool add = false, bool create = false, bool write = false, bool delete = false, bool delete_previous_version = false, bool tag = false, int days = 30, int hours = 0, int minutes = 0);
-
+    public Async.Task<Uri?> GetFileSasUrl(Container container, string name, StorageType storageType, BlobSasPermissions permissions, TimeSpan? duration = null);
+    Async.Task saveBlob(Container container, string v1, string v2, StorageType config);
+    Task<Guid> GetInstanceId();
 }
 
 public class Containers : IContainers
@@ -29,7 +32,7 @@ public Containers(ILogTracer log, IStorage storage, ICreds creds)
         _creds = creds;
         _armClient = creds.ArmClient;
     }
-    public async Task<IEnumerable<byte>?> GetBlob(Container container, string name, StorageType storageType)
+    public async Task<BinaryData?> GetBlob(Container container, string name, StorageType storageType)
     {
         var client = await FindContainer(container, storageType);
 
@@ -41,7 +44,7 @@ public Containers(ILogTracer log, IStorage storage, ICreds creds)
         try
         {
             return (await client.GetBlobClient(name).DownloadContentAsync())
-                .Value.Content.ToArray();
+                .Value.Content;
         }
         catch (RequestFailedException)
         {
@@ -85,9 +88,59 @@ private static Uri GetUrl(string accountName)
         return new Uri($"https://{accountName}.blob.core.windows.net/");
     }
 
-    public Uri GetFileSasUrl(Container container, string name, StorageType storageType, bool read = false, bool add = false, bool create = false, bool write = false, bool delete = false, bool delete_previous_version = false, bool tag = false, int days = 30, int hours = 0, int minutes = 0)
+    public async Async.Task<Uri?> GetFileSasUrl(Container container, string name, StorageType storageType, BlobSasPermissions permissions, TimeSpan? duration = null)
     {
-        throw new NotImplementedException();
+        var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container.ContainerName} - {storageType}");
+        var (accountName, accountKey) = _storage.GetStorageAccountNameAndKey(client.AccountName);
+
+        var (startTime, endTime) = SasTimeWindow(duration ?? TimeSpan.FromDays(30));
+
+        var sasBuilder = new BlobSasBuilder(permissions, endTime)
+        {
+            StartsOn = startTime,
+            BlobContainerName = container.ContainerName,
+            BlobName = name
+        };
+
+        var sasUrl = client.GetBlobClient(name).GenerateSasUri(sasBuilder);
+        return sasUrl;
+    }
+
+    public (DateTimeOffset, DateTimeOffset) SasTimeWindow(TimeSpan timeSpan)
+    {
+        // SAS URLs are valid 6 hours earlier, primarily to work around dev
+        // workstations having out-of-sync time.  Additionally, SAS URLs are stopped
+        // 15 minutes later than requested based on "Be careful with SAS start time"
+        // guidance.
+        // Ref: https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview
+
+        var SAS_START_TIME_DELTA = TimeSpan.FromHours(6);
+        var SAS_END_TIME_DELTA = TimeSpan.FromMinutes(6);
+
+        //    SAS_START_TIME_DELTA = datetime.timedelta(hours = 6)
+        //SAS_END_TIME_DELTA = datetime.timedelta(minutes = 15)
+
+        var now = DateTimeOffset.UtcNow;
+        var start = now - SAS_START_TIME_DELTA;
+        var expiry = now + timeSpan + SAS_END_TIME_DELTA;
+        return (start, expiry);
+    }
+
+    public async System.Threading.Tasks.Task saveBlob(Container container, string name, string data, StorageType storageType)
+    {
+        var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container.ContainerName} - {storageType}");
+
+        await client.UploadBlobAsync(name, new BinaryData(data));
+    }
+
+    public async Async.Task<Guid> GetInstanceId()
+    {
+        var blob = await GetBlob(new Container("base-config"), "instance_id", StorageType.Config);
+        if (blob == null)
+        {
+            throw new System.Exception("Blob Not Found");
+        }
+        return System.Guid.Parse(blob.ToString());
     }
 }
 

src/ApiService/ApiService/onefuzzlib/InstanceConfig.cs

@@ -15,12 +15,11 @@ public class ConfigOperations : Orm<InstanceConfig>, IConfigOperations
 {
     private readonly IEvents _events;
     private readonly ILogTracer _log;
-    private readonly IServiceConfig _config;
+
     public ConfigOperations(IStorage storage, IEvents events, ILogTracer log, IServiceConfig config) : base(storage, log, config)
     {
         _events = events;
         _log = log;
-        _config = config;
     }
 
     public async Task<InstanceConfig> Fetch()

src/ApiService/ApiService/onefuzzlib/Network.cs

@@ -25,12 +25,6 @@ public Network(string region, string group, string name, NetworkConfig networkCo
             _subnet = subnet;
         }
 
-
-        private static Guid GenerateGuidv5(Guid nameSpace, string name)
-        {
-            throw new NotImplementedException();
-        }
-
         public static async Async.Task<Network> Create(string region, ICreds creds, IConfigOperations configOperations, ISubnet subnet)
         {
             var group = creds.GetBaseResourceGroup();
@@ -50,7 +44,7 @@ public static async Async.Task<Network> Create(string region, ICreds creds, ICon
             }
             else
             {
-                var networkId = GenerateGuidv5(NETWORK_GUID_NAMESPACE, string.Join("|", networkConfig.AddressSpace, networkConfig.Subnet));
+                var networkId = Faithlife.Utility.GuidUtility.Create(NETWORK_GUID_NAMESPACE, string.Join("|", networkConfig.AddressSpace, networkConfig.Subnet), 5);
                 name = $"{region}-{networkId}";
             }
 

src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs

@@ -1,5 +1,6 @@
 using System.Text.Json;
 using ApiService.OneFuzzLib.Orm;
+using Azure.Storage.Sas;
 
 namespace Microsoft.OneFuzz.Service;
 
@@ -10,7 +11,6 @@ public interface INotificationOperations
 
 public class NotificationOperations : Orm<Notification>, INotificationOperations
 {
-    private ILogTracer _log;
     private IReports _reports;
     private ITaskOperations _taskOperations;
 
@@ -23,7 +23,7 @@ public class NotificationOperations : Orm<Notification>, INotificationOperations
     public NotificationOperations(ILogTracer log, IStorage storage, IReports reports, ITaskOperations taskOperations, IContainers containers, IQueue queue, IEvents events, IServiceConfig config)
         : base(storage, log, config)
     {
-        _log = log;
+
         _reports = reports;
         _taskOperations = taskOperations;
         _containers = containers;
@@ -76,9 +76,9 @@ public async Async.Task NewFiles(Container container, string filename, bool fail
         {
             if (containers.Contains(container.ContainerName))
             {
-                _log.Info($"queuing input {container.ContainerName} {filename} {task.TaskId}");
-                var url = _containers.GetFileSasUrl(container, filename, StorageType.Corpus, read: true, delete: true);
-                await _queue.SendMessage(task.TaskId.ToString(), System.Text.Encoding.UTF8.GetBytes(url.ToString()), StorageType.Corpus);
+                _logTracer.Info($"queuing input {container.ContainerName} {filename} {task.TaskId}");
+                var url = _containers.GetFileSasUrl(container, filename, StorageType.Corpus, BlobSasPermissions.Read | BlobSasPermissions.Delete);
+                await _queue.SendMessage(task.TaskId.ToString(), System.Text.Encoding.UTF8.GetBytes(url?.ToString() ?? ""), StorageType.Corpus);
             }
         }
 
@@ -125,7 +125,7 @@ public IAsyncEnumerable<Notification> GetNotifications(Container container)
             return await _taskOperations.GetByJobIdAndTaskId(report.CrashTestResult.NoReproReport.JobId, report.CrashTestResult.NoReproReport.TaskId);
         }
 
-        _log.Error($"unable to find crash_report or no repro entry for report: {JsonSerializer.Serialize(report)}");
+        _logTracer.Error($"unable to find crash_report or no repro entry for report: {JsonSerializer.Serialize(report)}");
         return null;
     }
 

src/ApiService/ApiService/onefuzzlib/Nsg.cs

@@ -72,8 +72,10 @@ public bool OkToDelete(HashSet<string> active_regions, string nsg_region, string
             return !active_regions.Contains(nsg_region) && nsg_region == nsg_name;
         }
 
-        // Returns True if deletion completed (thus resource not found) or successfully started.
-        // Returns False if failed to start deletion.
+        /// <summary>
+        /// Returns True if deletion completed (thus resource not found) or successfully started.
+        /// Returns False if failed to start deletion.
+        /// </summary>
         public async Async.Task<bool> StartDeleteNsg(string name)
         {
             _logTracer.Info($"deleting nsg: {name}");

src/ApiService/ApiService/onefuzzlib/ProxyForwardOperations.cs

@@ -0,0 +1,34 @@
+using ApiService.OneFuzzLib.Orm;
+
+namespace Microsoft.OneFuzz.Service;
+
+
+public interface IProxyForwardOperations : IOrm<ProxyForward>
+{
+    IAsyncEnumerable<ProxyForward> SearchForward(Guid? scalesetId = null, string? region = null, Guid? machineId = null, Guid? proxyId = null, int? dstPort = null);
+}
+
+
+public class ProxyForwardOperations : Orm<ProxyForward>, IProxyForwardOperations
+{
+    public ProxyForwardOperations(IStorage storage, ILogTracer logTracer, IServiceConfig config) : base(storage, logTracer, config)
+    {
+    }
+
+    public IAsyncEnumerable<ProxyForward> SearchForward(Guid? scalesetId = null, string? region = null, Guid? machineId = null, Guid? proxyId = null, int? dstPort = null)
+    {
+
+        var conditions =
+            new[] {
+                scalesetId != null ? $"scaleset_id eq '{scalesetId}'" : null,
+                region != null ? $"region eq '{region}'" : null ,
+                machineId != null ? $"machine_id eq '{machineId}'" : null ,
+                proxyId != null ? $"proxy_id eq '{proxyId}'" : null ,
+                dstPort != null ? $"dsp_port eq {dstPort }" : null ,
+            }.Where(x => x != null);
+
+        var filter = string.Join(" and ", conditions);
+
+        return QueryAsync(filter);
+    }
+}