Add restricted queuing (#2173)

* Add restricted queuing Add the ability to restrict queuing to only specific queues Also switch over to using those specific queues in open builds
dotnet · Mar 6, 2019 · 02fd06b · 02fd06b
1 parent b94a591
commit 02fd06b
Show file tree

Hide file tree

Showing 10 changed files with 50 additions and 24 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -25,7 +25,7 @@ jobs:
         # Will eventually change this to two BYOC pools.
         ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
           name: NetCorePublic-Int-Pool
-          queue: Windows.10.Amd64.Open
+          queue: BuildPool.Windows.10.Amd64.VS2017.Open
         ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
           name: dotnet-internal-temp
       variables:
@@ -96,7 +96,8 @@ jobs:
         container: LinuxContainer
         pool:
           ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
-            name: dnceng-linux-external-temp
+            name:  NetCorePublic-Int-Pool
+            queue: BuildPool.Ubuntu.1604.Amd64.Open
           ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
             name: dnceng-linux-internal-temp
         variables:

diff --git a/src/HelixPoolProvider/HelixPoolProvider/AllowableHelixQueues.cs b/src/HelixPoolProvider/HelixPoolProvider/AllowableHelixQueues.cs
@@ -8,6 +8,7 @@ public enum AllowableHelixQueues
     {
         Any,
         OnlyInternal,
-        NoInternal
+        NoInternal,
+        Specific
     }
 }
diff --git a/src/HelixPoolProvider/HelixPoolProvider/Config.cs b/src/HelixPoolProvider/HelixPoolProvider/Config.cs
@@ -8,6 +8,7 @@
 using Microsoft.Extensions.Logging;
 using System;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
 
 namespace Microsoft.DotNet.HelixPoolProvider
 {
@@ -32,6 +33,7 @@ public Config(IConfiguration config, ILoggerFactory loggerFactory)
         public string ApiAuthorizationPat => GetSecret(_configuration[$"{nameof(ApiAuthorizationPat)}-Key"]);
         public bool ApiAuthorizationPatIsConfigured => TryGetSecret(_configuration[$"{nameof(ApiAuthorizationPat)}-Key"], out string secretValue);
         public AllowableHelixQueues AllowedTargetQueues => Enum.Parse<AllowableHelixQueues>(_configuration[nameof(AllowedTargetQueues)]);
+        public string[] AllowedTargetQueueNames => _configuration.GetSection(nameof(AllowedTargetQueueNames)).Get<string[]>();
         public string HelixCreator => _configuration[nameof(HelixCreator)];
         public int TimeoutInMinutes => Int32.Parse(_configuration[nameof(TimeoutInMinutes)]);
         public string HelixEndpoint => _configuration[nameof(HelixEndpoint)];

diff --git a/src/HelixPoolProvider/HelixPoolProvider/Controllers/PoolProviderController.cs b/src/HelixPoolProvider/HelixPoolProvider/Controllers/PoolProviderController.cs
@@ -235,15 +235,25 @@ private bool IsAllowableQueue(QueueInfo queueInfo)
             {
                 return true;
             }
-
-            if (!queueInfo.IsInternalOnly.HasValue)
+            else if (_configuration.AllowedTargetQueues == AllowableHelixQueues.Specific)
             {
-                _logger.LogWarning($"Warning, unknown whether {queueInfo.QueueId} is internal only or not");
-                return false;
+                return _configuration.AllowedTargetQueueNames.Any(queueName => queueName.Equals(queueInfo.QueueId, StringComparison.OrdinalIgnoreCase));
+            }
+            else if (_configuration.AllowedTargetQueues == AllowableHelixQueues.NoInternal ||
+                     _configuration.AllowedTargetQueues == AllowableHelixQueues.OnlyInternal)
+            {
+                if (!queueInfo.IsInternalOnly.HasValue)
+                {
+                    _logger.LogWarning($"Warning, unknown whether {queueInfo.QueueId} is internal only or not");
+                    return false;
+                }
+                return (_configuration.AllowedTargetQueues == AllowableHelixQueues.NoInternal && !queueInfo.IsInternalOnly.Value) ||
+                       (_configuration.AllowedTargetQueues == AllowableHelixQueues.OnlyInternal && queueInfo.IsInternalOnly.Value);
+            }
+            else
+            {
+                throw new NotImplementedException($"Unexpected allowed target queue setting '{_configuration.AllowedTargetQueues}'");
             }
-
-            return (_configuration.AllowedTargetQueues == AllowableHelixQueues.NoInternal && !queueInfo.IsInternalOnly.Value) ||
-                (_configuration.AllowedTargetQueues == AllowableHelixQueues.OnlyInternal && queueInfo.IsInternalOnly.Value);
         }
 
         /// <summary>
@@ -411,6 +421,7 @@ public IActionResult GetInformation()
                 {
                     containerName = _configuration.ContainerName,
                     allowedTargetQueues = Enum.GetName(typeof(AllowableHelixQueues), _configuration.AllowedTargetQueues),
+                    allowedTargetQueueNames = _configuration.AllowedTargetQueues == AllowableHelixQueues.Specific ? _configuration.AllowedTargetQueueNames : null,
                     helixEndpoint = _configuration.HelixEndpoint,
                     timeoutInMinutes = _configuration.TimeoutInMinutes,
                     availableConnectionString = _configuration.ConnectionStringIsConfigured,

diff --git a/src/HelixPoolProvider/HelixPoolProvider/HelixJobCreator.cs b/src/HelixPoolProvider/HelixPoolProvider/HelixJobCreator.cs
@@ -91,7 +91,7 @@ public async Task<AgentInfoItem> CreateJob()
                 // Now that we have a valid queue, construct the Helix job on that queue
                 var job = await _api.Job.Define()
                     .WithSource($"agent/{_agentRequestItem.accountId}/")
-                    .WithType(string.Empty)
+                    .WithType($"byoc/{_configuration.HelixCreator}/")
                     .WithBuild("1.0")
                     .WithTargetQueue(_queueInfo.QueueId)
                     .WithCreator(_configuration.HelixCreator)

diff --git a/...vider/Properties/PublishProfiles/helixpoolprovider-dncenginternal-int - Web Deploy.pubxml b/...vider/Properties/PublishProfiles/helixpoolprovider-dncenginternal-int - Web Deploy.pubxml
@@ -6,15 +6,18 @@ by editing this MSBuild file. In order to learn more about this please visit htt
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <WebPublishMethod>MSDeploy</WebPublishMethod>
-    <ResourceId>/subscriptions/cab65fc3-d077-467d-931f-3932eabf36d3/resourcegroups/helixpoolprovider/providers/Microsoft.Web/sites/helixpoolprovider-dncenginternal-int</ResourceId>
+    <ResourceId>/subscriptions/cab65fc3-d077-467d-931f-3932eabf36d3/resourceGroups/helixpoolprovider/providers/Microsoft.Web/sites/helixpoolprovider-dncenginternal-int</ResourceId>
     <ResourceGroup>helixpoolprovider</ResourceGroup>
     <PublishProvider>AzureWebSite</PublishProvider>
     <LastUsedBuildConfiguration>Release</LastUsedBuildConfiguration>
     <LastUsedPlatform>Any CPU</LastUsedPlatform>
     <SiteUrlToLaunchAfterPublish>https://helixpoolprovider-dncenginternal-int.azurewebsites.net</SiteUrlToLaunchAfterPublish>
     <LaunchSiteAfterPublish>True</LaunchSiteAfterPublish>
     <ExcludeApp_Data>False</ExcludeApp_Data>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
     <ProjectGuid>5ed29e38-a65d-4123-bfed-799ada885f08</ProjectGuid>
+    <SelfContained>false</SelfContained>
+    <_IsPortable>true</_IsPortable>
     <MSDeployServiceURL>helixpoolprovider-dncenginternal-int.scm.azurewebsites.net:443</MSDeployServiceURL>
     <DeployIisAppPath>helixpoolprovider-dncenginternal-int</DeployIisAppPath>
     <RemoteSitePhysicalPath />
@@ -25,8 +28,5 @@ by editing this MSBuild file. In order to learn more about this please visit htt
     <_SavePWD>True</_SavePWD>
     <_DestinationType>AzureWebSite</_DestinationType>
     <InstallAspNetCoreSiteExtension>False</InstallAspNetCoreSiteExtension>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
-    <SelfContained>false</SelfContained>
-    <_IsPortable>true</_IsPortable>
   </PropertyGroup>
 </Project>
diff --git a/src/HelixPoolProvider/HelixPoolProvider/appsettings.dncenginternal-int.json b/src/HelixPoolProvider/HelixPoolProvider/appsettings.dncenginternal-int.json
@@ -1,11 +1,15 @@
 {
   "ContainerName": "internal-workitemstorage",
   "ConnectionString-Key": "https://helixstagingkv.vault.azure.net/secrets/helixpoolprovider-dnceng-PayloadConnectionString",
-  "ApiAuthorizationPat-Key": "https://helixstagingkv.vault.azure.net/secrets/helixpoolprovider-dnceng-ApiAuthorizationPat",
+  "ApiAuthorizationPat-Key": "https://buildagents.vault.azure.net/secrets/BotAccount-helix-buildagent-bot-helix-token",
   "SharedSecret-Key": "https://helixstagingkv.vault.azure.net/secrets/helixpoolprovider-dncenginternal-SharedSecret",
-  "AllowedTargetQueues": "OnlyInternal",
+  "AllowedTargetQueues": "Specific",
+  "AllowedTargetQueueNames": [
+    "BuildPool.Ubuntu.1604.Amd64",
+    "BuildPool.Windows.10.Amd64.VS2017"
+  ],
   "HelixCreator": "helixpoolprovider-dncengpublic-int",
-  "TimeoutInMinutes": 60,
+  "TimeoutInMinutes": 600,
   "HelixEndpoint": "https://helix.dot.net",
   "MaxParallelism": 1000,
   "Logging": {

diff --git a/src/HelixPoolProvider/HelixPoolProvider/appsettings.dncengpublic-int.json b/src/HelixPoolProvider/HelixPoolProvider/appsettings.dncengpublic-int.json
@@ -1,11 +1,15 @@
 {
   "ContainerName": "public-workitemstorage",
   "ConnectionString-Key": "https://helixstagingkv.vault.azure.net/secrets/helixpoolprovider-dnceng-PayloadConnectionString",
-  "ApiAuthorizationPat-Key": "https://helixstagingkv.vault.azure.net/secrets/helixpoolprovider-dnceng-ApiAuthorizationPat",
+  "ApiAuthorizationPat-Key": "https://buildagents.vault.azure.net/secrets/BotAccount-helix-buildagent-bot-helix-token",
   "SharedSecret-Key": "https://helixstagingkv.vault.azure.net/secrets/helixpoolprovider-dncengpublic-SharedSecret",
-  "AllowedTargetQueues": "NoInternal",
+  "AllowedTargetQueues": "Specific",
+  "AllowedTargetQueueNames": [
+    "BuildPool.Ubuntu.1604.Amd64.Open",
+    "BuildPool.Windows.10.Amd64.VS2017.Open"
+  ],
   "HelixCreator": "helixpoolprovider-dncengpublic-int",
-  "TimeoutInMinutes": 60,
+  "TimeoutInMinutes": 600,
   "HelixEndpoint": "https://helix.dot.net",
   "MaxParallelism": 1000,
   "Logging": {

diff --git a/src/HelixPoolProvider/HelixPoolProvider/configure-pool.ps1 b/src/HelixPoolProvider/HelixPoolProvider/configure-pool.ps1
@@ -38,7 +38,7 @@ if (-not $skipCreateCloud) {
 if (-not $skipCreatePool) {
 	try {
 		echo "Creating agent pool for cloud"
-		$result = Invoke-WebRequest -Headers $allHeaders -Method POST "$accountUrl/_apis/DistributedTask/pools?api-version=5.0-preview" -Body "{`"name`":`"$byocProviderName-Pool`",`"agentCloudId`":$agentCloudId}"
+		$result = Invoke-WebRequest -Headers $allHeaders -Method POST "$accountUrl/_apis/DistributedTask/pools?api-version=5.0-preview" -Body "{`"name`":`"$byocProviderName-Pool`",`"agentCloudId`":$agentCloudId,`"targetSize`":500}"
 		if ($result.StatusCode -ne 200) {
 			echo $result.Content
 			throw "Failed to create pool for agent cloud"

diff --git a/src/HelixPoolProvider/HelixPoolProvider/wwwroot/startupscripts/startagent-linux.sh b/src/HelixPoolProvider/HelixPoolProvider/wwwroot/startupscripts/startagent-linux.sh
@@ -3,7 +3,10 @@
 workspace_path=$1
 
 # Make the workspace path directory.
-mkdir $workspace_path
+sudo mkdir $workspace_path
+
+# Make it writeable
+sudo chmod -R 777 $workspace_path
 
 # copy .agent and .credentials files to workspace path
 cp -f -r $HELIX_CORRELATION_PAYLOAD/* $workspace_path
@@ -20,4 +23,4 @@ if [[ $lastexitcode -ne 2 ]]; then
 else
 	echo "Agent disconnected successfully, exiting"
 	exit 0
-fi
+fi