From 86d6d6596c309727ad23e20f357c1312ffbb6767 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 20:48:53 +0000 Subject: [PATCH 01/14] Adding New Default Image Config Value to IC. --- src/ApiService/ApiService/Functions/Scaleset.cs | 13 ++++++++++++- src/ApiService/ApiService/OneFuzzTypes/Model.cs | 4 ++++ src/ApiService/ApiService/OneFuzzTypes/Requests.cs | 2 +- src/ApiService/FunctionalTests/1f-api/Scaleset.cs | 2 +- src/api-service/__app__/scaleset/__init__.py | 10 +++++++++- src/cli/onefuzz/api.py | 4 ++-- src/pytypes/onefuzztypes/models.py | 6 ++++++ src/pytypes/onefuzztypes/requests.py | 2 +- 8 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/ApiService/ApiService/Functions/Scaleset.cs b/src/ApiService/ApiService/Functions/Scaleset.cs index bbdf987c05..0c892d2100 100644 --- a/src/ApiService/ApiService/Functions/Scaleset.cs +++ b/src/ApiService/ApiService/Functions/Scaleset.cs @@ -75,6 +75,17 @@ private async Task Post(HttpRequestData req) { context: "ScalesetCreate"); } + string image; + if (create.Image is null) { + if (pool.Os == Os.Windows) { + image = (await _context.ConfigOperations.Fetch()).DefaultWindowsVmImage; + } else { + image = (await _context.ConfigOperations.Fetch()).DefaultLinxuVmImage; + } + } else { + image = create.Image; + } + Region region; if (create.Region is null) { region = await _context.Creds.GetBaseRegion(); @@ -117,7 +128,7 @@ private async Task Post(HttpRequestData req) { Auth: await Auth.BuildAuth(_log), PoolName: create.PoolName, VmSku: create.VmSku, - Image: create.Image, + Image: image, Region: region, Size: create.Size, SpotInstances: create.SpotInstances, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index f39d4f88f4..8550afb055 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -329,6 +329,8 @@ public record InstanceConfig [DefaultValue(InitMethod.DefaultConstructor)] NetworkConfig NetworkConfig, [DefaultValue(InitMethod.DefaultConstructor)] NetworkSecurityGroupConfig ProxyNsgConfig, AzureVmExtensionConfig? Extensions, + string DefaultWindowsVmImage = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", + string DefaultLinxuVmImage = "Canonical:UbuntuServer:18.04-LTS:latest", string ProxyVmSku = "Standard_B2s", bool RequireAdminPrivileges = false, IDictionary? ApiAccessRules = null, @@ -343,6 +345,8 @@ public InstanceConfig(string instanceName) : this( new NetworkConfig(), new NetworkSecurityGroupConfig(), null, + "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", + "Canonical:UbuntuServer:18.04-LTS:latest", "Standard_B2s", false ) { } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs index 320470be7f..63488a4922 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs @@ -190,7 +190,7 @@ public record ProxyReset( public record ScalesetCreate( [property: Required] PoolName PoolName, [property: Required] string VmSku, - [property: Required] string Image, + string? Image, Region? Region, [property: Range(1, long.MaxValue), Required] long Size, [property: Required] bool SpotInstances, diff --git a/src/ApiService/FunctionalTests/1f-api/Scaleset.cs b/src/ApiService/FunctionalTests/1f-api/Scaleset.cs index 5998a1d458..f659ede4e6 100644 --- a/src/ApiService/FunctionalTests/1f-api/Scaleset.cs +++ b/src/ApiService/FunctionalTests/1f-api/Scaleset.cs @@ -73,7 +73,7 @@ public async Task, Error>> Get(Guid? id = null, str return IEnumerableResult(res); } - public async Task> Create(string poolName, int size, string? region = null, string vmSku = "Standard_D2s_v3", string image = Image_Ubuntu_20_04, bool spotInstance = false) { + public async Task> Create(string poolName, int size, string? region = null, string vmSku = "Standard_D2s_v3", string? image = null, bool spotInstance = false) { _output.WriteLine($"Creating scaleset in pool {poolName}, size: {size}"); var rootScalesetCreate = new JsonObject() diff --git a/src/api-service/__app__/scaleset/__init__.py b/src/api-service/__app__/scaleset/__init__.py index a3969559bd..560fe94770 100644 --- a/src/api-service/__app__/scaleset/__init__.py +++ b/src/api-service/__app__/scaleset/__init__.py @@ -78,6 +78,14 @@ def post(req: func.HttpRequest) -> func.HttpResponse: region = request.region + if request.image is None: + if pool.os == "windows": + image = instance_config.default_windows_vm_image + else: + image = instance_config.default_linux_vm_image + else: + image = request.image + if request.vm_sku not in list_available_skus(region): return not_ok( Error( @@ -97,7 +105,7 @@ def post(req: func.HttpRequest) -> func.HttpResponse: scaleset = Scaleset.create( pool_name=request.pool_name, vm_sku=request.vm_sku, - image=request.image, + image=image, region=region, size=request.size, spot_instances=request.spot_instances, diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 5ccc53c670..44debace71 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1427,9 +1427,9 @@ def create( if image is None: pool = self.onefuzz.pools.get(pool_name) if pool.os == enums.OS.linux: - image = DEFAULT_LINUX_IMAGE + image = "default_linux" elif pool.os == enums.OS.windows: - image = DEFAULT_WINDOWS_IMAGE + image = "default_windows" else: raise NotImplementedError diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index a136dc0e10..72859b2b32 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -880,6 +880,12 @@ class InstanceConfig(BaseModel): default_factory=NetworkSecurityGroupConfig ) extensions: Optional[AzureVmExtensionConfig] + default_windows_vm_image: str = Field( + default="MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest" + ) + default__linux_vm_image: str = Field( + default="Canonical:UbuntuServer:18.04-LTS:latest" + ) proxy_vm_sku: str = Field(default="Standard_B2s") api_access_rules: Optional[Dict[Endpoint, ApiAccessRule]] = None group_membership: Optional[Dict[PrincipalID, List[GroupId]]] = None diff --git a/src/pytypes/onefuzztypes/requests.py b/src/pytypes/onefuzztypes/requests.py index 82e8cc80b4..54bad0adec 100644 --- a/src/pytypes/onefuzztypes/requests.py +++ b/src/pytypes/onefuzztypes/requests.py @@ -182,7 +182,7 @@ class AutoScaleOptions(BaseModel): class ScalesetCreate(BaseRequest): pool_name: PoolName vm_sku: str - image: str + image: Optional[str] region: Optional[Region] size: int = Field(ge=1) spot_instances: bool From c2f96e8c0703609b05070518058d10ab7cfaee95 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 20:50:29 +0000 Subject: [PATCH 02/14] Removing forced image setting. --- src/cli/onefuzz/api.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 44debace71..a5d9cc25cb 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1424,15 +1424,6 @@ def create( if tags is None: tags = {} - if image is None: - pool = self.onefuzz.pools.get(pool_name) - if pool.os == enums.OS.linux: - image = "default_linux" - elif pool.os == enums.OS.windows: - image = "default_windows" - else: - raise NotImplementedError - auto_scale = requests.AutoScaleOptions( min=min_instances, max=max_size, From cd333a777ea7ba483f1f1dc0d886b443cf0e608d Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 21:01:07 +0000 Subject: [PATCH 03/14] Updating Webhook Events. --- docs/webhook_events.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/webhook_events.md b/docs/webhook_events.md index 34dcd8f47c..7a55320ec7 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -682,6 +682,8 @@ If webhook is set to have Event Grid message format then the payload will look a "allowed_aad_tenants": [ "00000000-0000-0000-0000-000000000000" ], + "default__linux_vm_image": "Canonical:UbuntuServer:18.04-LTS:latest", + "default_windows_vm_image": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", "network_config": { "address_space": "10.0.0.0/8", "subnet": "10.0.0.0/16" @@ -822,6 +824,16 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Api Access Rules", "type": "object" }, + "default__linux_vm_image": { + "default": "Canonical:UbuntuServer:18.04-LTS:latest", + "title": "Default Linux Vm Image", + "type": "string" + }, + "default_windows_vm_image": { + "default": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", + "title": "Default Windows Vm Image", + "type": "string" + }, "extensions": { "$ref": "#/definitions/AzureVmExtensionConfig" }, @@ -6046,6 +6058,16 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Api Access Rules", "type": "object" }, + "default__linux_vm_image": { + "default": "Canonical:UbuntuServer:18.04-LTS:latest", + "title": "Default Linux Vm Image", + "type": "string" + }, + "default_windows_vm_image": { + "default": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", + "title": "Default Windows Vm Image", + "type": "string" + }, "extensions": { "$ref": "#/definitions/AzureVmExtensionConfig" }, From 194797646b77c5c61fe176b9dac65d8c8fc8d72a Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 21:19:50 +0000 Subject: [PATCH 04/14] Removing typo. --- src/pytypes/onefuzztypes/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 72859b2b32..72c99d2595 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -883,7 +883,7 @@ class InstanceConfig(BaseModel): default_windows_vm_image: str = Field( default="MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest" ) - default__linux_vm_image: str = Field( + default_linux_vm_image: str = Field( default="Canonical:UbuntuServer:18.04-LTS:latest" ) proxy_vm_sku: str = Field(default="Standard_B2s") From c2d021971f19d1725ccee711defcdf391bd22e49 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 21:23:31 +0000 Subject: [PATCH 05/14] Updating webhook_events again. --- docs/webhook_events.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/webhook_events.md b/docs/webhook_events.md index 7a55320ec7..e521ca3a75 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -682,7 +682,7 @@ If webhook is set to have Event Grid message format then the payload will look a "allowed_aad_tenants": [ "00000000-0000-0000-0000-000000000000" ], - "default__linux_vm_image": "Canonical:UbuntuServer:18.04-LTS:latest", + "default_linux_vm_image": "Canonical:UbuntuServer:18.04-LTS:latest", "default_windows_vm_image": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", "network_config": { "address_space": "10.0.0.0/8", @@ -824,7 +824,7 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Api Access Rules", "type": "object" }, - "default__linux_vm_image": { + "default_linux_vm_image": { "default": "Canonical:UbuntuServer:18.04-LTS:latest", "title": "Default Linux Vm Image", "type": "string" @@ -6058,7 +6058,7 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Api Access Rules", "type": "object" }, - "default__linux_vm_image": { + "default_linux_vm_image": { "default": "Canonical:UbuntuServer:18.04-LTS:latest", "title": "Default Linux Vm Image", "type": "string" From 152076f17dfb1a2280ab66187ed80607b768cdaf Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 21:28:55 +0000 Subject: [PATCH 06/14] Syncing webhook events. --- docs/webhook_events.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/webhook_events.md b/docs/webhook_events.md index e521ca3a75..07fc1a0d9e 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -826,7 +826,7 @@ If webhook is set to have Event Grid message format then the payload will look a }, "default_linux_vm_image": { "default": "Canonical:UbuntuServer:18.04-LTS:latest", - "title": "Default Linux Vm Image", + "title": "Default Linux Vm Image", "type": "string" }, "default_windows_vm_image": { @@ -6060,7 +6060,7 @@ If webhook is set to have Event Grid message format then the payload will look a }, "default_linux_vm_image": { "default": "Canonical:UbuntuServer:18.04-LTS:latest", - "title": "Default Linux Vm Image", + "title": "Default Linux Vm Image", "type": "string" }, "default_windows_vm_image": { From 8139a443496395b444bc5e1cc56c417ab319c079 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 22:38:34 +0000 Subject: [PATCH 07/14] Fixing check for os type. --- src/api-service/__app__/scaleset/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api-service/__app__/scaleset/__init__.py b/src/api-service/__app__/scaleset/__init__.py index 560fe94770..90f39e7fc1 100644 --- a/src/api-service/__app__/scaleset/__init__.py +++ b/src/api-service/__app__/scaleset/__init__.py @@ -4,7 +4,7 @@ # Licensed under the MIT License. import azure.functions as func -from onefuzztypes.enums import ErrorCode, ScalesetState +from onefuzztypes.enums import ErrorCode, OS, ScalesetState from onefuzztypes.models import Error from onefuzztypes.requests import ( ScalesetCreate, @@ -79,7 +79,7 @@ def post(req: func.HttpRequest) -> func.HttpResponse: region = request.region if request.image is None: - if pool.os == "windows": + if pool.os == OS.windows: image = instance_config.default_windows_vm_image else: image = instance_config.default_linux_vm_image From 857712fa75dfe322ac5a32f1d688dcbe3c10a491 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 22:39:33 +0000 Subject: [PATCH 08/14] Fixing import. --- src/api-service/__app__/scaleset/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api-service/__app__/scaleset/__init__.py b/src/api-service/__app__/scaleset/__init__.py index 90f39e7fc1..32ca011c99 100644 --- a/src/api-service/__app__/scaleset/__init__.py +++ b/src/api-service/__app__/scaleset/__init__.py @@ -4,7 +4,7 @@ # Licensed under the MIT License. import azure.functions as func -from onefuzztypes.enums import ErrorCode, OS, ScalesetState +from onefuzztypes.enums import OS, ErrorCode, ScalesetState from onefuzztypes.models import Error from onefuzztypes.requests import ( ScalesetCreate, From 5de95d381e8088877d585b776a8bc3b9fda39b01 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 23:04:47 +0000 Subject: [PATCH 09/14] PR Suggestions. --- src/ApiService/ApiService/Functions/Scaleset.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ApiService/ApiService/Functions/Scaleset.cs b/src/ApiService/ApiService/Functions/Scaleset.cs index 0c892d2100..434c19d7cf 100644 --- a/src/ApiService/ApiService/Functions/Scaleset.cs +++ b/src/ApiService/ApiService/Functions/Scaleset.cs @@ -77,10 +77,11 @@ private async Task Post(HttpRequestData req) { string image; if (create.Image is null) { + var config = await _context.ConfigOperations.Fetch(); if (pool.Os == Os.Windows) { - image = (await _context.ConfigOperations.Fetch()).DefaultWindowsVmImage; + image = config.DefaultWindowsVmImage; } else { - image = (await _context.ConfigOperations.Fetch()).DefaultLinxuVmImage; + image = config.DefaultLinxuVmImage; } } else { image = create.Image; From d11a36a5e573796b1773adf00ddfc94ec169fd7a Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 23:47:06 +0000 Subject: [PATCH 10/14] Fix C# Model Typo. --- src/ApiService/ApiService/Functions/Scaleset.cs | 2 +- src/ApiService/ApiService/OneFuzzTypes/Model.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ApiService/ApiService/Functions/Scaleset.cs b/src/ApiService/ApiService/Functions/Scaleset.cs index 434c19d7cf..7736c29687 100644 --- a/src/ApiService/ApiService/Functions/Scaleset.cs +++ b/src/ApiService/ApiService/Functions/Scaleset.cs @@ -81,7 +81,7 @@ private async Task Post(HttpRequestData req) { if (pool.Os == Os.Windows) { image = config.DefaultWindowsVmImage; } else { - image = config.DefaultLinxuVmImage; + image = config.DefaultLinuxVmImage; } } else { image = create.Image; diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 8550afb055..a1104d6781 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -330,7 +330,7 @@ public record InstanceConfig [DefaultValue(InitMethod.DefaultConstructor)] NetworkSecurityGroupConfig ProxyNsgConfig, AzureVmExtensionConfig? Extensions, string DefaultWindowsVmImage = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", - string DefaultLinxuVmImage = "Canonical:UbuntuServer:18.04-LTS:latest", + string DefaultLinuxVmImage = "Canonical:UbuntuServer:18.04-LTS:latest", string ProxyVmSku = "Standard_B2s", bool RequireAdminPrivileges = false, IDictionary? ApiAccessRules = null, From 8a76f5489692c1b900dc26065f2e233ae78f741a Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 23:53:04 +0000 Subject: [PATCH 11/14] Removing other refs to images. --- src/ApiService/FunctionalTests/1f-api/Scaleset.cs | 3 --- src/cli/onefuzz/templates/__init__.py | 7 ------- 2 files changed, 10 deletions(-) diff --git a/src/ApiService/FunctionalTests/1f-api/Scaleset.cs b/src/ApiService/FunctionalTests/1f-api/Scaleset.cs index f659ede4e6..be685d5c6f 100644 --- a/src/ApiService/FunctionalTests/1f-api/Scaleset.cs +++ b/src/ApiService/FunctionalTests/1f-api/Scaleset.cs @@ -57,9 +57,6 @@ public Scaleset() { } public class ScalesetApi : ApiBase { - public const string Image_Ubuntu_20_04 = "Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest"; - public const string ImageWindows = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest"; - public ScalesetApi(Uri endpoint, Microsoft.OneFuzz.Service.Request request, ITestOutputHelper output) : base(endpoint, "/api/Scaleset", request, output) { } diff --git a/src/cli/onefuzz/templates/__init__.py b/src/cli/onefuzz/templates/__init__.py index 596887c9bb..979a0e131f 100644 --- a/src/cli/onefuzz/templates/__init__.py +++ b/src/cli/onefuzz/templates/__init__.py @@ -177,13 +177,6 @@ def upload_inputs_zip(self, path: File) -> None: self.containers[ContainerType.inputs], Directory(tmp_dir) ) - @classmethod - def get_image(_cls, platform: OS) -> str: - if platform == OS.linux: - return DEFAULT_LINUX_IMAGE - else: - return DEFAULT_WINDOWS_IMAGE - @classmethod def get_platform(_cls, target_exe: File) -> OS: with open(target_exe, "rb") as handle: From 5f04b7627bafbcc1324d8358d532dca4a3af27a2 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Wed, 21 Sep 2022 23:54:33 +0000 Subject: [PATCH 12/14] Removing remaining refs to images outside of models. --- src/cli/onefuzz/api.py | 3 --- src/cli/onefuzz/templates/__init__.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index a5d9cc25cb..17be517f31 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -48,9 +48,6 @@ ONE_HOUR_IN_SECONDS = 3600 -DEFAULT_LINUX_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest" -DEFAULT_WINDOWS_IMAGE = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest" - REPRO_SSH_FORWARD = "1337:127.0.0.1:1337" UUID_RE = r"^[a-f0-9]{8}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{12}\Z" diff --git a/src/cli/onefuzz/templates/__init__.py b/src/cli/onefuzz/templates/__init__.py index 979a0e131f..2c9e3a7929 100644 --- a/src/cli/onefuzz/templates/__init__.py +++ b/src/cli/onefuzz/templates/__init__.py @@ -16,8 +16,6 @@ from ..job_templates.job_monitor import JobMonitor ELF_MAGIC = b"\x7fELF" -DEFAULT_LINUX_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest" -DEFAULT_WINDOWS_IMAGE = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest" class StoppedEarly(Exception): From 70605838a5243c21a541deca3f6e5387d10dc0b3 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Thu, 22 Sep 2022 16:29:06 +0000 Subject: [PATCH 13/14] Removing hardcoded image values from tests. --- src/ApiService/FunctionalTests/Helpers.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ApiService/FunctionalTests/Helpers.cs b/src/ApiService/FunctionalTests/Helpers.cs index caf3e8bdfa..b6e28fd5c9 100644 --- a/src/ApiService/FunctionalTests/Helpers.cs +++ b/src/ApiService/FunctionalTests/Helpers.cs @@ -3,14 +3,13 @@ namespace FunctionalTests { public class Helpers { public static async Task<(Pool, Scaleset)> CreatePoolAndScaleset(PoolApi poolApi, ScalesetApi scalesetApi, string os = "linux", string? region = null, int numNodes = 2) { - var image = (os == "linux") ? ScalesetApi.Image_Ubuntu_20_04 : ScalesetApi.ImageWindows; var newPoolId = Guid.NewGuid().ToString(); var newPoolName = PoolApi.TestPoolPrefix + newPoolId; var newPool = await poolApi.Create(newPoolName, os); Assert.True(newPool.IsOk, $"failed to create new pool: {newPool.ErrorV}"); - var newScalesetResult = await scalesetApi.Create(newPool.OkV!.Name, numNodes, region: region, image: image); + var newScalesetResult = await scalesetApi.Create(newPool.OkV!.Name, numNodes, region: region); Assert.True(newScalesetResult.IsOk, $"failed to crate new scaleset: {newScalesetResult.ErrorV}"); var newScaleset = newScalesetResult.OkV!; From 838371ef5281fdf27e20c05615beef9e284bd91d Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Thu, 22 Sep 2022 20:42:41 +0000 Subject: [PATCH 14/14] Update Default Proxy and Repro Images. --- .../ApiService/onefuzzlib/ProxyOperations.cs | 4 ++-- .../ApiService/onefuzzlib/ReproOperations.cs | 17 ++++++++--------- src/api-service/__app__/onefuzzlib/proxy.py | 4 ++-- src/api-service/__app__/onefuzzlib/repro.py | 14 +++++++------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs b/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs index 35ddba772d..9191b2003d 100644 --- a/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs @@ -232,7 +232,7 @@ private static IEnumerable GetErrors(Proxy proxy, VirtualMachineData vmD public static Vm GetVm(Proxy proxy, InstanceConfig config) { var tags = config.VmssTags; string proxyVmSku; - const string PROXY_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest"; + string proxyImage = config.DefaultLinuxVmImage; if (config.ProxyVmSku is null) { proxyVmSku = "Standard_B2s"; } else { @@ -243,7 +243,7 @@ public static Vm GetVm(Proxy proxy, InstanceConfig config) { Name: $"proxy-{proxy.ProxyId:N}", Region: proxy.Region, Sku: proxyVmSku, - Image: PROXY_IMAGE, + Image: proxyImage, Auth: proxy.Auth, Tags: tags, Nsg: null diff --git a/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs b/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs index 89c3705d77..27951e513d 100644 --- a/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs @@ -30,16 +30,9 @@ public interface IReproOperations : IStatefulOrm { } public class ReproOperations : StatefulOrm, IReproOperations { - private static readonly Dictionary DEFAULT_OS = new() - { - { Os.Linux, "Canonical:UbuntuServer:18.04-LTS:latest" }, - { Os.Windows, "MicrosoftWindowsDesktop:Windows-10:20h2-pro:latest" } - }; const string DEFAULT_SKU = "Standard_DS1_v2"; - - public ReproOperations(ILogTracer log, IOnefuzzContext context) : base(log, context) { @@ -57,16 +50,22 @@ public async Async.Task GetVm(Repro repro, InstanceConfig config) { throw new Exception($"previous existing task missing: {repro.TaskId}"); } + Dictionary default_os = new() + { + { Os.Linux, config.DefaultLinuxVmImage }, + { Os.Windows, config.DefaultWindowsVmImage } + }; + var vmConfig = await taskOperations.GetReproVmConfig(task); if (vmConfig == null) { - if (!DEFAULT_OS.ContainsKey(task.Os)) { + if (!default_os.ContainsKey(task.Os)) { throw new NotSupportedException($"unsupport OS for repro {task.Os}"); } vmConfig = new TaskVm( await _context.Creds.GetBaseRegion(), DEFAULT_SKU, - DEFAULT_OS[task.Os], + default_os[task.Os], null ); } diff --git a/src/api-service/__app__/onefuzzlib/proxy.py b/src/api-service/__app__/onefuzzlib/proxy.py index 1ae03cb634..53660a68a6 100644 --- a/src/api-service/__app__/onefuzzlib/proxy.py +++ b/src/api-service/__app__/onefuzzlib/proxy.py @@ -43,7 +43,6 @@ from .orm import ORMMixin, QueryFilter from .proxy_forward import ProxyForward -PROXY_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest" PROXY_LOG_PREFIX = "scaleset-proxy: " PROXY_LIFESPAN = datetime.timedelta(days=7) @@ -70,6 +69,7 @@ def key_fields(cls) -> Tuple[str, Optional[str]]: return ("region", "proxy_id") def get_vm(self, config: InstanceConfig) -> VM: + config = InstanceConfig.fetch() sku = config.proxy_vm_sku tags = None if config.vm_tags: @@ -78,7 +78,7 @@ def get_vm(self, config: InstanceConfig) -> VM: name="proxy-%s" % base58.b58encode(self.proxy_id.bytes).decode(), region=self.region, sku=sku, - image=PROXY_IMAGE, + image=config.default_linux_vm_image, auth=self.auth, tags=tags, ) diff --git a/src/api-service/__app__/onefuzzlib/repro.py b/src/api-service/__app__/onefuzzlib/repro.py index 68f6a07726..621a623810 100644 --- a/src/api-service/__app__/onefuzzlib/repro.py +++ b/src/api-service/__app__/onefuzzlib/repro.py @@ -27,11 +27,6 @@ from .reports import get_report from .tasks.main import Task -DEFAULT_OS = { - OS.linux: "Canonical:UbuntuServer:18.04-LTS:latest", - OS.windows: "MicrosoftWindowsDesktop:Windows-10:20h2-pro:latest", -} - DEFAULT_SKU = "Standard_DS1_v2" @@ -56,14 +51,19 @@ def get_vm(self, config: InstanceConfig) -> VM: if isinstance(task, Error): raise Exception("previously existing task missing: %s" % self.task_id) + config = InstanceConfig.fetch() + default_os = { + OS.linux: config.default_linux_vm_image, + OS.windows: config.default_windows_vm_image, + } vm_config = task.get_repro_vm_config() if vm_config is None: # if using a pool without any scalesets defined yet, use reasonable defaults - if task.os not in DEFAULT_OS: + if task.os not in default_os: raise NotImplementedError("unsupported OS for repro %s" % task.os) vm_config = TaskVm( - region=get_base_region(), sku=DEFAULT_SKU, image=DEFAULT_OS[task.os] + region=get_base_region(), sku=DEFAULT_SKU, image=default_os[task.os] ) if self.auth is None: