From ccf97472b1a783b46cef72ab6a880935d2036948 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Fri, 18 Nov 2022 00:12:00 +0000 Subject: [PATCH 01/17] Remove Python App Function Deployment Code and Code. --- src/ApiService/ApiService/onefuzzlib/Creds.cs | 3 +- .../__app__/onefuzzlib/azure/creds.py | 11 +- src/cli/onefuzz/api.py | 12 -- src/cli/onefuzz/backend.py | 9 +- src/deployment/azuredeploy.bicep | 165 +++--------------- .../bicep-templates/autoscale-settings.bicep | 5 +- .../bicep-templates/function-settings.bicep | 19 +- .../bicep-templates/server-farms.bicep | 3 +- src/deployment/deploy.py | 146 +--------------- src/integration-tests/integration-test.py | 24 --- 10 files changed, 36 insertions(+), 361 deletions(-) diff --git a/src/ApiService/ApiService/onefuzzlib/Creds.cs b/src/ApiService/ApiService/onefuzzlib/Creds.cs index 69d82d5468..85b350e76c 100644 --- a/src/ApiService/ApiService/onefuzzlib/Creds.cs +++ b/src/ApiService/ApiService/onefuzzlib/Creds.cs @@ -106,8 +106,7 @@ public Async.Task GetBaseRegion() { } public Uri GetInstanceUrl() - // TODO: remove -net when promoted to main version - => new($"https://{GetInstanceName()}-net.azurewebsites.net"); + => new($"https://{GetInstanceName()}.azurewebsites.net"); public record ScaleSetIdentity(string principalId); diff --git a/src/api-service/__app__/onefuzzlib/azure/creds.py b/src/api-service/__app__/onefuzzlib/azure/creds.py index 7cedba4cb4..4342b3ade6 100644 --- a/src/api-service/__app__/onefuzzlib/azure/creds.py +++ b/src/api-service/__app__/onefuzzlib/azure/creds.py @@ -80,18 +80,9 @@ def get_instance_name() -> str: def get_instance_url() -> str: return "https://%s.azurewebsites.net" % get_instance_name() - -@cached -def use_dotnet_agent_functions() -> bool: - return os.environ.get("ONEFUZZ_USE_DOTNET_AGENT_FUNCTIONS") == "1" - - @cached def get_agent_instance_url() -> str: - if use_dotnet_agent_functions(): - return "https://%s-net.azurewebsites.net" % get_instance_name() - else: - return get_instance_url() + return get_instance_url() @cached diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index b091ded41e..916e88bfb1 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1796,8 +1796,6 @@ def __setup__( client_secret: Optional[str] = None, authority: Optional[str] = None, tenant_domain: Optional[str] = None, - _dotnet_endpoint: Optional[str] = None, - _dotnet_functions: Optional[List[str]] = None, ) -> None: if endpoint: @@ -1810,10 +1808,6 @@ def __setup__( self._backend.client_secret = client_secret if tenant_domain is not None: self._backend.config.tenant_domain = tenant_domain - if _dotnet_endpoint is not None: - self._backend.config.dotnet_endpoint = _dotnet_endpoint - if _dotnet_functions is not None: - self._backend.config.dotnet_functions = _dotnet_functions if self._backend.is_feature_enabled(PreviewFeature.job_templates.name): self.job_templates._load_cache() @@ -1857,8 +1851,6 @@ def config( client_id: Optional[str] = None, enable_feature: Optional[PreviewFeature] = None, tenant_domain: Optional[str] = None, - _dotnet_endpoint: Optional[str] = None, - _dotnet_functions: Optional[List[str]] = None, reset: Optional[bool] = None, ) -> BackendConfig: """Configure onefuzz CLI""" @@ -1889,10 +1881,6 @@ def config( self._backend.enable_feature(enable_feature.name) if tenant_domain is not None: self._backend.config.tenant_domain = tenant_domain - if _dotnet_endpoint is not None: - self._backend.config.dotnet_endpoint = _dotnet_endpoint - if _dotnet_functions is not None: - self._backend.config.dotnet_functions = _dotnet_functions self._backend.app = None self._backend.save_config() diff --git a/src/cli/onefuzz/backend.py b/src/cli/onefuzz/backend.py index 6d6a2cefd1..9c87784aef 100644 --- a/src/cli/onefuzz/backend.py +++ b/src/cli/onefuzz/backend.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +f#!/usr/bin/env python # # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -96,8 +96,6 @@ class BackendConfig(BaseModel): endpoint: Optional[str] features: Set[str] = Field(default_factory=set) tenant_domain: Optional[str] - dotnet_endpoint: Optional[str] - dotnet_functions: Optional[List[str]] class Backend: @@ -301,10 +299,7 @@ def request( params: Optional[Any] = None, _retry_on_auth_failure: bool = True, ) -> Response: - if self.config.dotnet_functions and path in self.config.dotnet_functions: - endpoint = self.config.dotnet_endpoint - else: - endpoint = self.config.endpoint + endpoint = self.config.endpoint if not endpoint: raise Exception("endpoint not configured") diff --git a/src/deployment/azuredeploy.bicep b/src/deployment/azuredeploy.bicep index b267e55ef0..bab13c3943 100644 --- a/src/deployment/azuredeploy.bicep +++ b/src/deployment/azuredeploy.bicep @@ -26,14 +26,9 @@ param workbookData object ]) param diagnosticsLogLevel string = 'Verbose' -param use_dotnet_agent_functions bool - var log_retention = 30 var tenantId = subscription().tenantId -var python_functions_disabled = '0' -var dotnet_functions_disabled = '1' - var scaleset_identity = '${name}-scalesetid' var StorageBlobDataReader = '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' @@ -80,29 +75,16 @@ module operationalInsights 'bicep-templates/operational-insights.bicep' = { } } -module linuxServerFarm 'bicep-templates/server-farms.bicep' = { - name: 'linux-server-farm' +module serverFarm 'bicep-templates/server-farms.bicep' = { + name: 'server-farm' params: { server_farm_name: name owner: owner location: location - use_windows: false - create: true - } -} - -module dotNetServerFarm 'bicep-templates/server-farms.bicep' = { - name: (enable_remote_debugging) ? 'windows-server-farm' : 'same-linux-server-farm' - params: { - server_farm_name: (enable_remote_debugging) ? '${name}-net' : name - owner: owner - location: location use_windows: enable_remote_debugging - create: enable_remote_debugging } } - var keyVaultName = 'of-kv-${uniqueString(resourceGroup().id)}' resource keyVault 'Microsoft.KeyVault/vaults@2021-10-01' = { name: keyVaultName @@ -120,7 +102,7 @@ resource keyVault 'Microsoft.KeyVault/vaults@2021-10-01' = { } accessPolicies: [ { - objectId: pythonFunction.outputs.principalId + objectId: function.outputs.principalId tenantId: tenantId permissions: { secrets: [ @@ -169,33 +151,15 @@ module autoscaleSettings 'bicep-templates/autoscale-settings.bicep' = { name: 'autoscaleSettings' params: { location: location - server_farm_id: linuxServerFarm.outputs.id + server_farm_id: serverFarm.outputs.id owner: owner workspaceId: operationalInsights.outputs.workspaceId logRetention: log_retention - autoscale_name: 'onefuzz-autoscale-${uniqueString(resourceGroup().id)}' - create_new: true + autoscale_name: 'onefuzz-autoscale-${uniqueString(resourceGroup().id)}' function_diagnostics_settings_name: 'functionDiagnosticSettings' } } -module autoscaleSettingsNet 'bicep-templates/autoscale-settings.bicep' = { - name: 'autoscaleSettingsNet' - params: { - location: location - server_farm_id: dotNetServerFarm.outputs.id - owner: owner - workspaceId: operationalInsights.outputs.workspaceId - logRetention: log_retention - autoscale_name: (enable_remote_debugging) ? 'onefuzz-autoscale-${uniqueString(resourceGroup().id)}-net' : 'onefuzz-autoscale-${uniqueString(resourceGroup().id)}' - create_new: enable_remote_debugging - function_diagnostics_settings_name: (enable_remote_debugging) ? 'functionDiagnosticSettings' : 'functionDiagnosticsSettingsNet' - } - dependsOn: [ - autoscaleSettings - ] -} - module eventGrid 'bicep-templates/event-grid.bicep' = { name: 'event-grid' params:{ @@ -211,15 +175,15 @@ module eventGrid 'bicep-templates/event-grid.bicep' = { // try to make role assignments to deploy as late as possible in order to have principalId ready resource roleAssigmentsPy 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = [for r in roleAssignmentsParams: { - name: guid('${resourceGroup().id}${r.suffix}-python') + name: guid('${resourceGroup().id}${r.suffix}-cs') properties: { roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${r.role}' - principalId: pythonFunction.outputs.principalId + principalId: function.outputs.principalId } dependsOn: [ eventGrid keyVault - linuxServerFarm + serverFarm ] }] @@ -233,14 +197,14 @@ resource roleAssigmentsNet 'Microsoft.Authorization/roleAssignments@2020-10-01-p dependsOn: [ eventGrid keyVault - dotNetServerFarm + serverFarm ] }] // try to make role assignments to deploy as late as possible in order to have principalId ready resource readBlobUserAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { - name: guid('${resourceGroup().id}-user_managed_idenity_read_blob') + name: guid('${resourceGroup().id}-user_managed_idenity_read_blob-cs') properties: { roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${StorageBlobDataReader}' principalId: reference(scalesetIdentity.id, scalesetIdentity.apiVersion, 'Full').properties.principalId @@ -248,30 +212,29 @@ resource readBlobUserAssignment 'Microsoft.Authorization/roleAssignments@2020-10 dependsOn: [ eventGrid keyVault - linuxServerFarm - dotNetServerFarm - ] + serverFarm + ] } -module pythonFunction 'bicep-templates/function.bicep' = { - name: 'pythonFunction' +module function 'bicep-templates/function.bicep' = { + name: 'function' params: { name: name - linux_fx_version: 'Python|3.8' + linux_fx_version: 'DOTNET-ISOLATED|6.0' app_logs_sas_url: storage.outputs.FuncSasUrlBlobAppLogs app_func_audiences: app_func_audiences app_func_issuer: app_func_issuer - + client_id: clientId diagnostics_log_level: diagnosticsLogLevel location: location log_retention: log_retention owner: owner - server_farm_id: linuxServerFarm.outputs.id - client_id: clientId - use_windows: false - enable_remote_debugging: false + server_farm_id: serverFarm.outputs.id + + use_windows: enable_remote_debugging + enable_remote_debugging: enable_remote_debugging } } @@ -289,15 +252,15 @@ module netFunction 'bicep-templates/function.bicep' = { location: location log_retention: log_retention owner: owner - server_farm_id: dotNetServerFarm.outputs.id + server_farm_id: serverFarm.outputs.id use_windows: enable_remote_debugging enable_remote_debugging: enable_remote_debugging } } -module pythonFunctionSettings 'bicep-templates/function-settings.bicep' = { - name: 'pythonFunctionSettings' +module functionSettings 'bicep-templates/function-settings.bicep' = { + name: 'functionSettings' params: { name: name owner: owner @@ -314,50 +277,10 @@ module pythonFunctionSettings 'bicep-templates/function-settings.bicep' = { keyvault_name: keyVaultName monitor_account_name: operationalInsights.outputs.monitorAccountName multi_tenant_domain: multi_tenant_domain - functions_disabled: python_functions_disabled - use_dotnet_agent_functions: use_dotnet_agent_functions enable_profiler: false - all_function_names: [ - 'agent_can_schedule' //0 - 'agent_commands' //1 - 'agent_events' //2 - 'agent_registration' //3 - 'containers' //4 - 'download' //5 - 'info' //6 - 'instance_config' //7 - 'jobs' //8 - 'job_templates' //9 - 'job_templates_manage' //10 - 'negotiate' //11 - 'node' //12 - 'node_add_ssh_key' //13 - 'notifications' //14 - 'pool' //15 - 'proxy' //16 - 'queue_file_changes' //17 - 'queue_node_heartbeat' //18 - 'queue_proxy_update' //19 - 'queue_signalr_events' //20 - 'queue_task_heartbeat' //21 - 'queue_updates' //22 - 'queue_webhooks' //23 - 'repro_vms' //24 - 'scaleset' //25 - 'tasks' //26 - 'timer_daily' //27 - 'timer_proxy' //28 - 'timer_repro' //29 - 'timer_retention' //30 - 'timer_tasks' //31 - 'timer_workers' //32 - 'webhooks' //33 - 'webhooks_logs' //34 - 'webhooks_ping' //35 - ] } dependsOn: [ - pythonFunction + function ] } @@ -380,47 +303,7 @@ module netFunctionSettings 'bicep-templates/function-settings.bicep' = { keyvault_name: keyVaultName monitor_account_name: operationalInsights.outputs.monitorAccountName multi_tenant_domain: multi_tenant_domain - functions_disabled: dotnet_functions_disabled - use_dotnet_agent_functions: false // this doesn’t do anything on the .NET service enable_profiler: enable_profiler - all_function_names: [ - 'AgentCanSchedule' //0 - 'AgentCommands' //1 - 'AgentEvents' //2 - 'AgentRegistration' //3 - 'Containers' //4 - 'Download' //5 - 'Info' //6 - 'InstanceConfig' //7 - 'Jobs' //8 - 'JobTemplates' //9 - 'JobTemplatesManage' //10 - 'Negotiate' //11 - 'Node' //12 - 'NodeAddSshKey' //13 - 'Notifications' //14 - 'Pool' //15 - 'Proxy' //16 - 'QueueFileChanges' //17 - 'QueueNodeHeartbeat' //18 - 'QueueProxyUpdate' //19 - 'QueueSignalrEvents' //20 - 'QueueTaskHeartbeat' //21 - 'QueueUpdates' //22 - 'QueueWebhooks' //23 - 'ReproVms' //24 - 'Scaleset' //25 - 'Tasks' //26 - 'TimerDaily' //27 - 'TimerProxy' //28 - 'TimerRepro' //29 - 'TimerRetention' //30 - 'TimerTasks' //31 - 'TimerWorkers' //32 - 'Webhooks' //33 - 'WebhooksLogs' //34 - 'WebhooksPing' //35 - ] } dependsOn: [ netFunction diff --git a/src/deployment/bicep-templates/autoscale-settings.bicep b/src/deployment/bicep-templates/autoscale-settings.bicep index bd581ed803..e380be272e 100644 --- a/src/deployment/bicep-templates/autoscale-settings.bicep +++ b/src/deployment/bicep-templates/autoscale-settings.bicep @@ -5,10 +5,9 @@ param workspaceId string param logRetention int param autoscale_name string param function_diagnostics_settings_name string -param create_new bool -resource autoscaleSettings 'Microsoft.Insights/autoscalesettings@2015-04-01' = if (create_new) { +resource autoscaleSettings 'Microsoft.Insights/autoscalesettings@2015-04-01' = { name: autoscale_name location: location properties: { @@ -72,7 +71,7 @@ resource autoscaleSettings 'Microsoft.Insights/autoscalesettings@2015-04-01' = i } } -resource functionDiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (create_new) { +resource functionDiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { name: function_diagnostics_settings_name scope: autoscaleSettings properties: { diff --git a/src/deployment/bicep-templates/function-settings.bicep b/src/deployment/bicep-templates/function-settings.bicep index 4395e8fc85..51bb891032 100644 --- a/src/deployment/bicep-templates/function-settings.bicep +++ b/src/deployment/bicep-templates/function-settings.bicep @@ -26,29 +26,14 @@ param monitor_account_name string param functions_worker_runtime string param functions_extension_version string -param functions_disabled string -param use_dotnet_agent_functions bool - -param all_function_names array - param enable_profiler bool -var disabledFunctionName = 'disabledFunctions-${functions_worker_runtime}' - var telemetry = 'd7a73cf4-5a1a-4030-85e1-e5b25867e45a' resource function 'Microsoft.Web/sites@2021-02-01' existing = { name: name } -module disabledFunctions 'function-settings-disabled-apps.bicep' = { - name: disabledFunctionName - params:{ - functions_disabled_setting: functions_disabled - allFunctions: all_function_names - } -} - var enable_profilers = enable_profiler ? { APPINSIGHTS_PROFILERFEATURE_VERSION : '1.0.0' DiagnosticServices_EXTENSION_VERSION: '~3' @@ -78,6 +63,6 @@ resource functionSettings 'Microsoft.Web/sites/config@2021-03-01' = { ONEFUZZ_KEYVAULT: keyvault_name ONEFUZZ_OWNER: owner ONEFUZZ_CLIENT_SECRET: client_secret - ONEFUZZ_USE_DOTNET_AGENT_FUNCTIONS: use_dotnet_agent_functions ? '1' : '0' - }, disabledFunctions.outputs.appSettings, enable_profilers) + ONEFUZZ_USE_DOTNET_AGENT_FUNCTIONS: '1' + }, enable_profilers) } diff --git a/src/deployment/bicep-templates/server-farms.bicep b/src/deployment/bicep-templates/server-farms.bicep index 9ed38a96cb..d918d3c95f 100644 --- a/src/deployment/bicep-templates/server-farms.bicep +++ b/src/deployment/bicep-templates/server-farms.bicep @@ -2,11 +2,10 @@ param server_farm_name string param owner string param location string param use_windows bool -param create bool var kind = (use_windows) ? 'app' : 'linux' -resource serverFarms 'Microsoft.Web/serverfarms@2022-03-01' = if (create) { +resource serverFarms 'Microsoft.Web/serverfarms@2022-03-01' = { name: server_farm_name location: location kind: kind diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index 892a6f9d9b..ceec47c910 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -95,12 +95,7 @@ ) DOTNET_APPLICATION_SUFFIX = "-net" -DOTNET_AGENT_FUNCTIONS = [ - "agent_can_schedule", - "agent_commands", - "agent_events", - "agent_registration", -] + logger = logging.getLogger("deploy") @@ -145,7 +140,6 @@ def __init__( client_id: Optional[str], client_secret: Optional[str], app_zip: str, - app_net_zip: str, tools: str, instance_specific: str, third_party: str, @@ -159,8 +153,6 @@ def __init__( subscription_id: Optional[str], admins: List[UUID], allowed_aad_tenants: List[UUID], - enable_dotnet: List[str], - use_dotnet_agent_functions: bool, cli_app_id: str, auto_create_cli_app: bool, host_dotnet_on_windows: bool, @@ -173,7 +165,6 @@ def __init__( self.owner = owner self.nsg_config = nsg_config self.app_zip = app_zip - self.app_net_zip = app_net_zip self.tools = tools self.instance_specific = instance_specific self.third_party = third_party @@ -195,8 +186,6 @@ def __init__( self.arm_template = bicep_to_arm(bicep_template) - self.enable_dotnet = enable_dotnet - self.use_dotnet_agent_functions = use_dotnet_agent_functions self.cli_app_id = cli_app_id self.auto_create_cli_app = auto_create_cli_app self.host_dotnet_on_windows = host_dotnet_on_windows @@ -1122,7 +1111,6 @@ def deploy_app(self) -> None: "functionapp", "publish", self.application_name, - "--python", "--no-build", ], env=dict(os.environ, CLI_DEBUG="1"), @@ -1142,9 +1130,9 @@ def deploy_app(self) -> None: raise error def deploy_dotnet_app(self) -> None: - logger.info("deploying function app %s ", self.app_net_zip) + logger.info("deploying function app %s ", self.app_zip) with tempfile.TemporaryDirectory() as tmpdirname: - with zipfile.ZipFile(self.app_net_zip, "r") as zip_ref: + with zipfile.ZipFile(self.app_zip, "r") as zip_ref: func = shutil.which("func") assert func is not None @@ -1178,99 +1166,6 @@ def deploy_dotnet_app(self) -> None: if error is not None: raise error - def enable_dotnet_func(self) -> None: - if self.enable_dotnet: - - def expand_agent(f: str) -> List[str]: - # 'agent' is permitted as a shortcut for the agent functions - if f == "agent": - return DOTNET_AGENT_FUNCTIONS - else: - return [f] - - enable_dotnet = itertools.chain.from_iterable( - map(expand_agent, self.enable_dotnet) - ) - - python_settings = [] - dotnet_settings = [] - - for function_name in enable_dotnet: - format_name = function_name.split("_") - dotnet_name = "".join(x.title() for x in format_name) - # keep the python versions of http function to allow the service to be backward compatible - # with older version of the CLI and the agents - if function_name.startswith("queue_") or function_name.startswith( - "timer_" - ): - logger.info(f"disabling PYTHON function: {function_name}") - disable_python = "1" - else: - logger.info(f"enabling PYTHON function: {function_name}") - disable_python = "0" - - python_settings.append( - f"AzureWebJobs.{function_name}.Disabled={disable_python}" - ) - - # enable dotnet function - logger.info(f"enabling DOTNET function: {dotnet_name}") - dotnet_settings.append(f"AzureWebJobs.{dotnet_name}.Disabled=0") - - func = shutil.which("az") - assert func is not None - - max_tries = 5 - error: Optional[subprocess.CalledProcessError] = None - for i in range(max_tries): - try: - logger.info("updating Python settings") - subprocess.check_output( - [ - func, - "functionapp", - "config", - "appsettings", - "set", - "--name", - self.application_name, - "--resource-group", - self.application_name, - "--settings", - ] - + python_settings, - env=dict(os.environ, CLI_DEBUG="1"), - ) - logger.info("updating .NET settings") - subprocess.check_output( - [ - func, - "functionapp", - "config", - "appsettings", - "set", - "--name", - self.application_name + DOTNET_APPLICATION_SUFFIX, - "--resource-group", - self.application_name, - "--settings", - ] - + dotnet_settings, - env=dict(os.environ, CLI_DEBUG="1"), - ) - break - except subprocess.CalledProcessError as err: - error = err - if i + 1 < max_tries: - logger.debug("func failure error: %s", err) - logger.warning( - "unable to update settings, waiting 60 seconds and trying again" - ) - time.sleep(60) - - if error is not None: - raise error - def update_registration(self) -> None: if not self.create_registration: return @@ -1334,7 +1229,6 @@ def main() -> None: ("dotnet-api", Client.deploy_dotnet_app), ("export_appinsights", Client.add_log_export), ("update_registration", Client.update_registration), - ("enable_dotnet", Client.enable_dotnet_func), ] formatter = argparse.ArgumentDefaultsHelpFormatter @@ -1359,12 +1253,6 @@ def main() -> None: parser.add_argument( "--app-zip", type=arg_file, - default="api-service.zip", - help="(default: %(default)s)", - ) - parser.add_argument( - "--app-net-zip", - type=arg_file, default="api-service-net.zip", help="(default: %(default)s)", ) @@ -1445,20 +1333,6 @@ def main() -> None: nargs="*", help="Set additional AAD tenants beyond the tenant the app is deployed in", ) - parser.add_argument( - "--enable_dotnet", - type=str, - nargs="+", - default=[], - help="Provide a space-seperated list of python function names to disable " - "their functions and enable corresponding dotnet functions in the Azure " - "Function App deployment", - ) - parser.add_argument( - "--use_dotnet_agent_functions", - action="store_true", - help="Tell the OneFuzz agent to use the dotnet endpoint", - ) parser.add_argument( "--cli_app_id", type=str, @@ -1497,7 +1371,6 @@ def main() -> None: client_id=args.client_id, client_secret=args.client_secret, app_zip=args.app_zip, - app_net_zip=args.app_net_zip, tools=args.tools, instance_specific=args.instance_specific, third_party=args.third_party, @@ -1511,8 +1384,6 @@ def main() -> None: subscription_id=args.subscription_id, admins=args.set_admins, allowed_aad_tenants=args.allowed_aad_tenants or [], - enable_dotnet=args.enable_dotnet, - use_dotnet_agent_functions=args.use_dotnet_agent_functions, cli_app_id=args.cli_app_id, auto_create_cli_app=args.auto_create_cli_app, host_dotnet_on_windows=args.host_dotnet_on_windows, @@ -1527,17 +1398,6 @@ def main() -> None: logging.getLogger("deploy").setLevel(logging.INFO) - if args.use_dotnet_agent_functions: - # validate that the agent functions are actually enabled - if not ( - "agent" in args.enable_dotnet - or all(map(lambda f: f in args.enable_dotnet, DOTNET_AGENT_FUNCTIONS)) - ): - logger.error( - "If --use_dotnet_agent_functions is set, all agent functions must be enabled (--enable_dotnet agent)." - ) - sys.exit(1) - if args.rbac_only: logger.warning( "'rbac_only' specified. The deployment will execute " diff --git a/src/integration-tests/integration-test.py b/src/integration-tests/integration-test.py index 1b9eac9bec..e1ed2f9759 100755 --- a/src/integration-tests/integration-test.py +++ b/src/integration-tests/integration-test.py @@ -988,16 +988,12 @@ def check_jobs( poll: bool = False, stop_on_complete_check: bool = False, job_ids: List[UUID] = [], - dotnet_endpoint: Optional[str] = None, - dotnet_functions: Optional[List[str]] = None, ) -> None: self.onefuzz.__setup__( endpoint=endpoint, client_id=client_id, client_secret=client_secret, authority=authority, - _dotnet_endpoint=dotnet_endpoint, - _dotnet_functions=dotnet_functions, ) tester = TestOnefuzz(self.onefuzz, self.logger, test_id) result = tester.check_jobs( @@ -1015,16 +1011,12 @@ def check_repros( client_secret: Optional[str], authority: Optional[str] = None, job_ids: List[UUID] = [], - dotnet_endpoint: Optional[str] = None, - dotnet_functions: Optional[List[str]] = None, ) -> None: self.onefuzz.__setup__( endpoint=endpoint, client_id=client_id, client_secret=client_secret, authority=authority, - _dotnet_endpoint=dotnet_endpoint, - _dotnet_functions=dotnet_functions, ) tester = TestOnefuzz(self.onefuzz, self.logger, test_id) launch_result, repros = tester.launch_repro(job_ids=job_ids) @@ -1043,8 +1035,6 @@ def setup( region: Optional[Region] = None, os_list: List[OS] = [OS.linux, OS.windows], test_id: Optional[UUID] = None, - dotnet_endpoint: Optional[str] = None, - dotnet_functions: Optional[List[str]] = None, ) -> None: if test_id is None: test_id = uuid4() @@ -1080,8 +1070,6 @@ def launch( targets: List[str] = list(TARGETS.keys()), test_id: Optional[UUID] = None, duration: int = 1, - dotnet_endpoint: Optional[str] = None, - dotnet_functions: Optional[List[str]] = None, ) -> None: if test_id is None: test_id = uuid4() @@ -1116,16 +1104,12 @@ def cleanup( authority: Optional[str], client_id: Optional[str], client_secret: Optional[str], - dotnet_endpoint: Optional[str] = None, - dotnet_functions: Optional[List[str]] = None, ) -> None: self.onefuzz.__setup__( endpoint=endpoint, client_id=client_id, client_secret=client_secret, authority=authority, - _dotnet_endpoint=dotnet_endpoint, - _dotnet_functions=dotnet_functions, ) tester = TestOnefuzz(self.onefuzz, self.logger, test_id=test_id) tester.cleanup() @@ -1138,16 +1122,12 @@ def check_logs( authority: Optional[str] = None, client_id: Optional[str], client_secret: Optional[str], - dotnet_endpoint: Optional[str] = None, - dotnet_functions: Optional[List[str]] = None, ) -> None: self.onefuzz.__setup__( endpoint=endpoint, client_id=client_id, client_secret=client_secret, authority=authority, - _dotnet_endpoint=dotnet_endpoint, - _dotnet_functions=dotnet_functions, ) tester = TestOnefuzz(self.onefuzz, self.logger, test_id=test_id) tester.check_logs_for_errors() @@ -1162,8 +1142,6 @@ def check_results( skip_repro: bool = False, test_id: UUID, job_ids: List[UUID] = [], - dotnet_endpoint: Optional[str] = None, - dotnet_functions: Optional[List[str]] = None, ) -> None: self.check_jobs( @@ -1207,8 +1185,6 @@ def test( targets: List[str] = list(TARGETS.keys()), skip_repro: bool = False, duration: int = 1, - dotnet_endpoint: Optional[str] = None, - dotnet_functions: Optional[List[str]] = None, ) -> None: success = True From 9f066b9f0b7abeda72c8e0a3727332d0543a86c8 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Fri, 18 Nov 2022 00:21:06 +0000 Subject: [PATCH 02/17] Updating yml and zip names. --- .github/workflows/ci.yml | 12 ++++++------ src/deployment/deploy.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c011216ea1..30f5b5b632 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -272,9 +272,9 @@ jobs: echo ${GITHUB_SHA} | tee onefuzzlib/git.version python -m pip install --upgrade pip pip install --target="./.python_packages/lib/site-packages" -r ./requirements.txt ${GITHUB_WORKSPACE}/artifacts/sdk/onefuzztypes-*.whl - zip -r api-service.zip . + zip -r api-service-py.zip . mkdir -p ${GITHUB_WORKSPACE}/artifacts/service - cp api-service.zip ${GITHUB_WORKSPACE}/artifacts/service + cp api-service-py.zip ${GITHUB_WORKSPACE}/artifacts/service - uses: actions/upload-artifact@v3 with: name: build-artifacts @@ -357,9 +357,9 @@ jobs: cd src/ApiService/ApiService/ mv az-local.settings.json bin/Release/net6.0/local.settings.json cd bin/Release/net6.0/ - zip -r api-service-net.zip . + zip -r api-service.zip . mkdir -p ${GITHUB_WORKSPACE}/artifacts/service-net - cp api-service-net.zip ${GITHUB_WORKSPACE}/artifacts/service-net + cp api-service.zip ${GITHUB_WORKSPACE}/artifacts/service-net - uses: actions/upload-artifact@v3 with: name: build-artifacts @@ -491,8 +491,8 @@ jobs: cp artifacts/agent-Linux/onefuzz-task src/deployment/tools/linux/ cp artifacts/proxy/onefuzz-proxy-manager src/deployment/tools/linux/ - cp artifacts/service/api-service.zip src/deployment - cp artifacts/service-net/api-service-net.zip src/deployment + cp artifacts/service/api-service-py.zip src/deployment + cp artifacts/service-net/api-service.zip src/deployment cp -r artifacts/third-party src/deployment echo $GITHUB_RUN_ID | tee src/deployment/.build.id echo $GITHUB_SHA | tee src/deployment/.sha diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index ceec47c910..8734aa211d 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -1253,7 +1253,7 @@ def main() -> None: parser.add_argument( "--app-zip", type=arg_file, - default="api-service-net.zip", + default="api-service.zip", help="(default: %(default)s)", ) parser.add_argument( From 91b491eaba78d948c134ea90899496f0989cf437 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 19:01:27 +0000 Subject: [PATCH 03/17] Fixing ci.yml. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd09706b8d..fc49d32150 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -357,8 +357,8 @@ jobs: run: | ./src/ci/get-version.sh > src/deployment/VERSION cd src/ApiService/ApiService/ - mv az-local.settings.json bin/Release/net6.0/local.settings.json - cd bin/Release/net6.0/ + mv az-local.settings.json bin/Release/net7.0/local.settings.json + cd bin/Release/net7.0/ zip -r api-service.zip . mkdir -p ${GITHUB_WORKSPACE}/artifacts/service-net cp api-service.zip ${GITHUB_WORKSPACE}/artifacts/service-net From 27511efcde7c14525164affeec7a0472bb68beb9 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 19:08:07 +0000 Subject: [PATCH 04/17] Typo. --- src/cli/onefuzz/backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/onefuzz/backend.py b/src/cli/onefuzz/backend.py index 9c87784aef..43d98aa8a0 100644 --- a/src/cli/onefuzz/backend.py +++ b/src/cli/onefuzz/backend.py @@ -1,4 +1,4 @@ -f#!/usr/bin/env python +#!/usr/bin/env python # # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. From 4ff358f71523223e3e9dbb6735984fdb8e893dea Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 19:21:02 +0000 Subject: [PATCH 05/17] Format --- src/api-service/__app__/onefuzzlib/azure/creds.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api-service/__app__/onefuzzlib/azure/creds.py b/src/api-service/__app__/onefuzzlib/azure/creds.py index 4342b3ade6..e4f0009500 100644 --- a/src/api-service/__app__/onefuzzlib/azure/creds.py +++ b/src/api-service/__app__/onefuzzlib/azure/creds.py @@ -80,6 +80,7 @@ def get_instance_name() -> str: def get_instance_url() -> str: return "https://%s.azurewebsites.net" % get_instance_name() + @cached def get_agent_instance_url() -> str: return get_instance_url() From 7993b399694eb24344bef6e840d9661d3370e0ee Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 19:33:49 +0000 Subject: [PATCH 06/17] Trying to remove python service. --- .github/workflows/ci.yml | 97 +++++++++---------- .../bicep-templates/function-settings.bicep | 1 - 2 files changed, 48 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 151ef7713d..a224cd91af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -252,54 +252,54 @@ jobs: with: name: build-artifacts path: artifacts - service: - needs: - - onefuzztypes - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - run: src/ci/set-versions.sh - - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - uses: actions/download-artifact@v3 - with: - name: build-artifacts - path: artifacts - - name: Build Service - run: | - cd src/api-service/__app__ - echo ${GITHUB_RUN_ID} | tee onefuzzlib/build.id - echo ${GITHUB_SHA} | tee onefuzzlib/git.version - python -m pip install --upgrade pip - pip install --target="./.python_packages/lib/site-packages" -r ./requirements.txt ${GITHUB_WORKSPACE}/artifacts/sdk/onefuzztypes-*.whl - zip -r api-service-py.zip . - mkdir -p ${GITHUB_WORKSPACE}/artifacts/service - cp api-service-py.zip ${GITHUB_WORKSPACE}/artifacts/service - - uses: actions/upload-artifact@v3 - with: - name: build-artifacts - path: artifacts - - name: lint - run: | - cd src/api-service - rm -rf __app__/.python_packages - python -m pip install --upgrade pip - pip install ${GITHUB_WORKSPACE}/artifacts/sdk/onefuzztypes-*.whl - pip install -r __app__/requirements.txt - pip install -r requirements-dev.txt - pytest tests - flake8 . - bandit -r ./__app__/ - black ./__app__/ ./tests --check - isort --profile black ./__app__/ ./tests --check - mypy __app__ ./tests - - # set a minimum confidence to ignore known false positives - vulture --min-confidence 61 __app__ - - ../ci/disable-py-cache.sh - mypy __app__ ./tests + # service: + # needs: + # - onefuzztypes + # runs-on: ubuntu-20.04 + # steps: + # - uses: actions/checkout@v3 + # - run: src/ci/set-versions.sh + # - uses: actions/setup-python@v4 + # with: + # python-version: 3.8 + # - uses: actions/download-artifact@v3 + # with: + # name: build-artifacts + # path: artifacts + # - name: Build Service + # run: | + # cd src/api-service/__app__ + # echo ${GITHUB_RUN_ID} | tee onefuzzlib/build.id + # echo ${GITHUB_SHA} | tee onefuzzlib/git.version + # python -m pip install --upgrade pip + # pip install --target="./.python_packages/lib/site-packages" -r ./requirements.txt ${GITHUB_WORKSPACE}/artifacts/sdk/onefuzztypes-*.whl + # zip -r api-service-py.zip . + # mkdir -p ${GITHUB_WORKSPACE}/artifacts/service + # cp api-service-py.zip ${GITHUB_WORKSPACE}/artifacts/service + # - uses: actions/upload-artifact@v3 + # with: + # name: build-artifacts + # path: artifacts + # - name: lint + # run: | + # cd src/api-service + # rm -rf __app__/.python_packages + # python -m pip install --upgrade pip + # pip install ${GITHUB_WORKSPACE}/artifacts/sdk/onefuzztypes-*.whl + # pip install -r __app__/requirements.txt + # pip install -r requirements-dev.txt + # pytest tests + # flake8 . + # bandit -r ./__app__/ + # black ./__app__/ ./tests --check + # isort --profile black ./__app__/ ./tests --check + # mypy __app__ ./tests + + # # set a minimum confidence to ignore known false positives + # vulture --min-confidence 61 __app__ + + # ../ci/disable-py-cache.sh + # mypy __app__ ./tests service-net: runs-on: ubuntu-22.04 steps: @@ -503,7 +503,6 @@ jobs: cp artifacts/agent-Linux/onefuzz-task src/deployment/tools/linux/ cp artifacts/proxy/onefuzz-proxy-manager src/deployment/tools/linux/ - cp artifacts/service/api-service-py.zip src/deployment cp artifacts/service-net/api-service.zip src/deployment cp -r artifacts/third-party src/deployment echo $GITHUB_RUN_ID | tee src/deployment/.build.id diff --git a/src/deployment/bicep-templates/function-settings.bicep b/src/deployment/bicep-templates/function-settings.bicep index 51bb891032..0487964690 100644 --- a/src/deployment/bicep-templates/function-settings.bicep +++ b/src/deployment/bicep-templates/function-settings.bicep @@ -63,6 +63,5 @@ resource functionSettings 'Microsoft.Web/sites/config@2021-03-01' = { ONEFUZZ_KEYVAULT: keyvault_name ONEFUZZ_OWNER: owner ONEFUZZ_CLIENT_SECRET: client_secret - ONEFUZZ_USE_DOTNET_AGENT_FUNCTIONS: '1' }, enable_profilers) } From 2fbe25d5e864b64b2d55caa935be1db505781cc1 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 19:34:58 +0000 Subject: [PATCH 07/17] Updating directories. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a224cd91af..0beee8aac5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -300,7 +300,7 @@ jobs: # ../ci/disable-py-cache.sh # mypy __app__ ./tests - service-net: + service: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -360,8 +360,8 @@ jobs: mv az-local.settings.json bin/Release/net7.0/local.settings.json cd bin/Release/net7.0/ zip -r api-service.zip . - mkdir -p ${GITHUB_WORKSPACE}/artifacts/service-net - cp api-service.zip ${GITHUB_WORKSPACE}/artifacts/service-net + mkdir -p ${GITHUB_WORKSPACE}/artifacts/service + cp api-service.zip ${GITHUB_WORKSPACE}/artifacts/service - uses: actions/upload-artifact@v3 with: name: build-artifacts @@ -503,7 +503,7 @@ jobs: cp artifacts/agent-Linux/onefuzz-task src/deployment/tools/linux/ cp artifacts/proxy/onefuzz-proxy-manager src/deployment/tools/linux/ - cp artifacts/service-net/api-service.zip src/deployment + cp artifacts/service/api-service.zip src/deployment cp -r artifacts/third-party src/deployment echo $GITHUB_RUN_ID | tee src/deployment/.build.id echo $GITHUB_SHA | tee src/deployment/.sha From c580b52b5bc175a884a37b191b8b562b26772c33 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 19:47:42 +0000 Subject: [PATCH 08/17] Removing flag. --- src/deployment/deploy.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index aad3fe9f78..39cdcdb7c5 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -689,7 +689,6 @@ def deploy_template(self) -> None: "signedExpiry": {"value": expiry}, "multi_tenant_domain": multi_tenant_domain, "workbookData": {"value": self.workbook_data}, - "use_dotnet_agent_functions": {"value": self.use_dotnet_agent_functions}, "enable_remote_debugging": {"value": self.host_dotnet_on_windows}, "enable_profiler": {"value": self.enable_profiler}, } @@ -1149,8 +1148,6 @@ def deploy_dotnet_app(self) -> None: "publish", self.application_name + DOTNET_APPLICATION_SUFFIX, "--no-build", - "--dotnet-version", - "7.0", ], env=dict(os.environ, CLI_DEBUG="1"), cwd=tmpdirname, @@ -1255,7 +1252,7 @@ def main() -> None: parser.add_argument( "--app-zip", type=arg_file, - default="api-service.zip", + default="api-service-net.zip", help="(default: %(default)s)", ) parser.add_argument( From 02f4a22356a7afd954f8a06cfcb5a25bd6288556 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 19:54:37 +0000 Subject: [PATCH 09/17] Format. --- src/deployment/deploy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index 39cdcdb7c5..876d9ce6f4 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -4,7 +4,6 @@ # Licensed under the MIT License. import argparse -import itertools import json import logging import os From 00bd34e23fdab596c50b12e86742496f9b2ea7a1 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 20:19:06 +0000 Subject: [PATCH 10/17] Fixng api-service-net ref. --- src/deployment/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index 876d9ce6f4..d8c75d8125 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -1251,7 +1251,7 @@ def main() -> None: parser.add_argument( "--app-zip", type=arg_file, - default="api-service-net.zip", + default="api-service.zip", help="(default: %(default)s)", ) parser.add_argument( From 9176ad36fae5ecad39bb0a3c350136a4f19bd13a Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 20:54:37 +0000 Subject: [PATCH 11/17] Re-add requirement. --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0beee8aac5..2fe14ae1d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -301,6 +301,8 @@ jobs: # ../ci/disable-py-cache.sh # mypy __app__ ./tests service: + needs: + - onefuzztypes runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 From db3c35285f6b4ec060c506af2356bbc30051c3fc Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 20:59:07 +0000 Subject: [PATCH 12/17] Fixing refs in bicep. --- src/deployment/azuredeploy.bicep | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/deployment/azuredeploy.bicep b/src/deployment/azuredeploy.bicep index 69f56c100c..8be830a275 100644 --- a/src/deployment/azuredeploy.bicep +++ b/src/deployment/azuredeploy.bicep @@ -221,7 +221,7 @@ module function 'bicep-templates/function.bicep' = { name: 'function' params: { name: name - linux_fx_version: 'DOTNET-ISOLATED|6.0' + linux_fx_version: 'DOTNET-ISOLATED|7.0' app_logs_sas_url: storage.outputs.FuncSasUrlBlobAppLogs app_func_audiences: app_func_audiences @@ -264,8 +264,8 @@ module functionSettings 'bicep-templates/function-settings.bicep' = { params: { name: name owner: owner - functions_worker_runtime: 'python' - functions_extension_version: '~3' + functions_worker_runtime: 'dotnet-isolated' + functions_extension_version: '~4' instance_name: name app_insights_app_id: operationalInsights.outputs.appInsightsAppId app_insights_key: operationalInsights.outputs.appInsightsInstrumentationKey @@ -277,7 +277,7 @@ module functionSettings 'bicep-templates/function-settings.bicep' = { keyvault_name: keyVaultName monitor_account_name: operationalInsights.outputs.monitorAccountName multi_tenant_domain: multi_tenant_domain - enable_profiler: false + enable_profiler: enable_profiler } dependsOn: [ function @@ -290,13 +290,13 @@ module netFunctionSettings 'bicep-templates/function-settings.bicep' = { params: { owner: owner name: '${name}-net' - signal_r_connection_string: signalR.outputs.connectionString - app_insights_app_id: operationalInsights.outputs.appInsightsAppId - app_insights_key: operationalInsights.outputs.appInsightsInstrumentationKey functions_worker_runtime: 'dotnet-isolated' functions_extension_version: '~4' instance_name: name + app_insights_app_id: operationalInsights.outputs.appInsightsAppId + app_insights_key: operationalInsights.outputs.appInsightsInstrumentationKey client_secret: clientSecret + signal_r_connection_string: signalR.outputs.connectionString func_sas_url: storage.outputs.FuncSasUrl func_storage_resource_id: storage.outputs.FuncId fuzz_storage_resource_id: storage.outputs.FuzzId From 156246c36a718a598905e44c181e95cc97b0053a Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 21:57:01 +0000 Subject: [PATCH 13/17] Specifying version. --- src/deployment/deploy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index d8c75d8125..67174c1b95 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -1110,6 +1110,8 @@ def deploy_app(self) -> None: "publish", self.application_name, "--no-build", + "--dotnet-version", + "7.0", ], env=dict(os.environ, CLI_DEBUG="1"), cwd=tmpdirname, @@ -1147,6 +1149,8 @@ def deploy_dotnet_app(self) -> None: "publish", self.application_name + DOTNET_APPLICATION_SUFFIX, "--no-build", + "--dotnet-version", + "7.0", ], env=dict(os.environ, CLI_DEBUG="1"), cwd=tmpdirname, From 9a5b68c64c3a548be43214b9ddf83051ca91c502 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 21 Nov 2022 22:23:40 +0000 Subject: [PATCH 14/17] Removing dotnet refs in integration tests. --- src/integration-tests/integration-test.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/integration-tests/integration-test.py b/src/integration-tests/integration-test.py index e1ed2f9759..d06ecff456 100755 --- a/src/integration-tests/integration-test.py +++ b/src/integration-tests/integration-test.py @@ -1039,9 +1039,6 @@ def setup( if test_id is None: test_id = uuid4() self.logger.info("launching test_id: %s", test_id) - self.logger.info( - "dotnet configuration: %s, %s", dotnet_endpoint, dotnet_functions - ) def try_setup(data: Any) -> None: self.onefuzz.__setup__( @@ -1049,8 +1046,6 @@ def try_setup(data: Any) -> None: client_id=client_id, client_secret=client_secret, authority=authority, - _dotnet_endpoint=dotnet_endpoint, - _dotnet_functions=dotnet_functions, ) retry(self.logger, try_setup, "trying to configure") @@ -1081,8 +1076,6 @@ def try_setup(data: Any) -> None: client_id=client_id, client_secret=client_secret, authority=authority, - _dotnet_endpoint=dotnet_endpoint, - _dotnet_functions=dotnet_functions, ) retry(self.logger, try_setup, "trying to configure") @@ -1153,8 +1146,6 @@ def check_results( poll=True, stop_on_complete_check=True, job_ids=job_ids, - dotnet_endpoint=dotnet_endpoint, - dotnet_functions=dotnet_functions, ) if skip_repro: @@ -1167,8 +1158,6 @@ def check_results( client_id=client_id, client_secret=client_secret, job_ids=job_ids, - dotnet_endpoint=dotnet_endpoint, - dotnet_functions=dotnet_functions, ) def test( @@ -1198,8 +1187,6 @@ def try_setup(data: Any) -> None: client_id=client_id, client_secret=client_secret, authority=authority, - _dotnet_endpoint=dotnet_endpoint, - _dotnet_functions=dotnet_functions, ) retry(self.logger, try_setup, "trying to configure") @@ -1234,8 +1221,6 @@ def try_setup(data: Any) -> None: client_id=client_id, client_secret=client_secret, authority=authority, - dotnet_endpoint=dotnet_endpoint, - dotnet_functions=dotnet_functions, ) except Exception as e: self.logger.error("testing failed: %s", repr(e)) From 2ace829e3c9175ad7b0dbe6d83da04d6a6b00543 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Tue, 22 Nov 2022 01:33:02 +0000 Subject: [PATCH 15/17] Updating role assignment naming convention. --- src/deployment/azuredeploy.bicep | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/deployment/azuredeploy.bicep b/src/deployment/azuredeploy.bicep index 8be830a275..247dfb29f2 100644 --- a/src/deployment/azuredeploy.bicep +++ b/src/deployment/azuredeploy.bicep @@ -174,8 +174,8 @@ module eventGrid 'bicep-templates/event-grid.bicep' = { } // try to make role assignments to deploy as late as possible in order to have principalId ready -resource roleAssigmentsPy 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = [for r in roleAssignmentsParams: { - name: guid('${resourceGroup().id}${r.suffix}-cs') +resource roleAssignments 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = [for r in roleAssignmentsParams: { + name: guid('${resourceGroup().id}${r.suffix}-onefuzz') properties: { roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${r.role}' principalId: function.outputs.principalId @@ -188,8 +188,8 @@ resource roleAssigmentsPy 'Microsoft.Authorization/roleAssignments@2020-10-01-pr }] // try to make role assignments to deploy as late as possible in order to have principalId ready -resource roleAssigmentsNet 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = [for r in roleAssignmentsParams: { - name: guid('${resourceGroup().id}${r.suffix}-net') +resource roleAssignmentsNet 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = [for r in roleAssignmentsParams: { + name: guid('${resourceGroup().id}${r.suffix}-onefuzz-net') properties: { roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${r.role}' principalId: netFunction.outputs.principalId @@ -204,7 +204,7 @@ resource roleAssigmentsNet 'Microsoft.Authorization/roleAssignments@2020-10-01-p // try to make role assignments to deploy as late as possible in order to have principalId ready resource readBlobUserAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { - name: guid('${resourceGroup().id}-user_managed_idenity_read_blob-cs') + name: guid('${resourceGroup().id}-user_managed_idenity_read_blob') properties: { roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${StorageBlobDataReader}' principalId: reference(scalesetIdentity.id, scalesetIdentity.apiVersion, 'Full').properties.principalId From 98cbd34991dc813296fa0444d1e205d3e2df752f Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Tue, 22 Nov 2022 17:54:26 +0000 Subject: [PATCH 16/17] Adding ignore. --- .github/workflows/ci.yml | 48 ---------------------------------------- src/deny.toml | 1 + 2 files changed, 1 insertion(+), 48 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fe14ae1d7..00ea662fa0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -252,54 +252,6 @@ jobs: with: name: build-artifacts path: artifacts - # service: - # needs: - # - onefuzztypes - # runs-on: ubuntu-20.04 - # steps: - # - uses: actions/checkout@v3 - # - run: src/ci/set-versions.sh - # - uses: actions/setup-python@v4 - # with: - # python-version: 3.8 - # - uses: actions/download-artifact@v3 - # with: - # name: build-artifacts - # path: artifacts - # - name: Build Service - # run: | - # cd src/api-service/__app__ - # echo ${GITHUB_RUN_ID} | tee onefuzzlib/build.id - # echo ${GITHUB_SHA} | tee onefuzzlib/git.version - # python -m pip install --upgrade pip - # pip install --target="./.python_packages/lib/site-packages" -r ./requirements.txt ${GITHUB_WORKSPACE}/artifacts/sdk/onefuzztypes-*.whl - # zip -r api-service-py.zip . - # mkdir -p ${GITHUB_WORKSPACE}/artifacts/service - # cp api-service-py.zip ${GITHUB_WORKSPACE}/artifacts/service - # - uses: actions/upload-artifact@v3 - # with: - # name: build-artifacts - # path: artifacts - # - name: lint - # run: | - # cd src/api-service - # rm -rf __app__/.python_packages - # python -m pip install --upgrade pip - # pip install ${GITHUB_WORKSPACE}/artifacts/sdk/onefuzztypes-*.whl - # pip install -r __app__/requirements.txt - # pip install -r requirements-dev.txt - # pytest tests - # flake8 . - # bandit -r ./__app__/ - # black ./__app__/ ./tests --check - # isort --profile black ./__app__/ ./tests --check - # mypy __app__ ./tests - - # # set a minimum confidence to ignore known false positives - # vulture --min-confidence 61 __app__ - - # ../ci/disable-py-cache.sh - # mypy __app__ ./tests service: needs: - onefuzztypes diff --git a/src/deny.toml b/src/deny.toml index 8fa38dc088..c9fc3989f2 100644 --- a/src/deny.toml +++ b/src/deny.toml @@ -17,6 +17,7 @@ yanked = "deny" ignore = [ "RUSTSEC-2022-0048", # xml-rs is unmaintained "RUSTSEC-2021-0139", # ansi_term is unmaintained + "RUSTSEC-2021-0145", # waiting for clap upgrade ] [bans] From fcac906f60bae1ba6799c1855be08a03daa5f663 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Tue, 22 Nov 2022 11:42:19 -0800 Subject: [PATCH 17/17] Update src/deny.toml Co-authored-by: George Pollard --- src/deny.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deny.toml b/src/deny.toml index c9fc3989f2..7097444d30 100644 --- a/src/deny.toml +++ b/src/deny.toml @@ -17,7 +17,7 @@ yanked = "deny" ignore = [ "RUSTSEC-2022-0048", # xml-rs is unmaintained "RUSTSEC-2021-0139", # ansi_term is unmaintained - "RUSTSEC-2021-0145", # waiting for clap upgrade + "RUSTSEC-2021-0145", # waiting for clap upgrade; we are unaffected by the bug (no custom allocator) ] [bans]