make SC user agent pool configurable (#871)

* user agent pool settings are now configurable * increase settings for CS-PR as they will run multiple CS instances at the same time Signed-off-by: Gerd Oberlechner <goberlec@redhat.com>
Azure · Nov 22, 2024 · 5c801ff · 5c801ff
1 parent 243aee9
commit 5c801ff
Show file tree

Hide file tree

Showing 10 changed files with 110 additions and 7 deletions.
diff --git a/config/Makefile b/config/Makefile
@@ -1,7 +1,11 @@
 materialize:
+	@echo "Materializing public cloud int env of config/config.yaml"
 	@../templatize.sh dev > public-cloud-dev.json
+	@echo "Materializing public cloud cs-pr env of config/config.yaml"
 	@../templatize.sh cs-pr > public-cloud-cs-pr.json
+	@echo "Materializing public cloud personal dev env of config/config.yaml"
 	@USER=tst ../templatize.sh personal-dev > public-cloud-personal-dev.json
+	@echo "Materializing public cloud int env of config/config.msft.yaml"
 	@CONFIG_FILE=../config/config.msft.yaml ../templatize.sh int > public-cloud-msft-int.json
 .PHONY: materialize
 

diff --git a/config/config.msft.yaml b/config/config.msft.yaml
@@ -136,8 +136,16 @@ clouds:
       int:
         # this is the MSFT INT environment
         defaults:
-            # OIDC
+          # OIDC
           oidcStorageAccountName: arohcpoidcint{{ .ctx.regionShort }}
+          # SVC
+          svc:
+            userAgentPool:
+              minCount: 1
+              maxCount: 3
+              vmSize: 'Standard_D2s_v3'
+              osDiskSizeGB: 32
+              azCount: 3
           # MC
           mgmt:
             # MGMTM AKS nodepools - big enough for 2 HCPs

diff --git a/config/config.schema.json b/config/config.schema.json
@@ -539,6 +539,34 @@
               "kvSoftDelete"
             ]
           },
+          "userAgentPool": {
+            "type": "object",
+            "properties": {
+              "azCount": {
+                "type": "number"
+              },
+              "maxCount": {
+                "type": "number"
+              },
+              "minCount": {
+                "type": "number"
+              },
+              "osDiskSizeGB": {
+                "type": "number"
+              },
+              "vmSize": {
+                "type": "string"
+              }
+            },
+            "additionalProperties": false,
+            "required": [
+              "azCount",
+              "maxCount",
+              "minCount",
+              "osDiskSizeGB",
+              "vmSize"
+            ]
+          },
           "rg": {
             "type": "string"
           },
@@ -550,7 +578,8 @@
         "required": [
           "etcd",
           "rg",
-          "subscription"
+          "subscription",
+          "userAgentPool"
         ]
       },
       "svcAcrName": {

diff --git a/config/config.yaml b/config/config.yaml
@@ -153,6 +153,12 @@ clouds:
         subscription: ARO Hosted Control Planes (EA Subscription 1)
         etcd:
           kvSoftDelete: false
+        userAgentPool:
+          minCount: 1
+          maxCount: 3
+          vmSize: 'Standard_D2s_v3'
+          osDiskSizeGB: 32
+          azCount: 3
       mgmt:
         # MGMTM AKS nodepools - big enough for 2 HCPs
         systemAgentPool:
@@ -207,8 +213,14 @@ clouds:
       cs-pr:
         # this is the cluster service PR check and full cycle test environment
         defaults:
-          # MGMTM AKS nodepools - big enough for multiple HCPs
+          svc:
+            # MC AKS nodepools
+            # big enough for multiple CS instances during PR checks
+            userAgentPool:
+              minCount: 2
+              maxCount: 12
           mgmt:
+            # MC AKS nodepools - big enough for multiple HCPs
             userAgentPool:
               minCount: 2
               maxCount: 12

diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json
@@ -136,7 +136,14 @@
       "kvSoftDelete": false
     },
     "rg": "hcp-underlay-cspr-svc",
-    "subscription": "ARO Hosted Control Planes (EA Subscription 1)"
+    "subscription": "ARO Hosted Control Planes (EA Subscription 1)",
+    "userAgentPool": {
+      "azCount": 3,
+      "maxCount": 12,
+      "minCount": 2,
+      "osDiskSizeGB": 32,
+      "vmSize": "Standard_D2s_v3"
+    }
   },
   "svcAcrName": "arohcpsvcdev",
   "vnetAddressPrefix": "10.128.0.0/14"

diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json
@@ -136,7 +136,14 @@
       "kvSoftDelete": false
     },
     "rg": "hcp-underlay-dev-svc",
-    "subscription": "ARO Hosted Control Planes (EA Subscription 1)"
+    "subscription": "ARO Hosted Control Planes (EA Subscription 1)",
+    "userAgentPool": {
+      "azCount": 3,
+      "maxCount": 3,
+      "minCount": 1,
+      "osDiskSizeGB": 32,
+      "vmSize": "Standard_D2s_v3"
+    }
   },
   "svcAcrName": "arohcpsvcdev",
   "vnetAddressPrefix": "10.128.0.0/14"

diff --git a/config/public-cloud-msft-int.json b/config/public-cloud-msft-int.json
@@ -136,7 +136,14 @@
       "kvSoftDelete": true
     },
     "rg": "hcp-underlay-westus3-svc",
-    "subscription": "hcp-westus3"
+    "subscription": "hcp-westus3",
+    "userAgentPool": {
+      "azCount": 3,
+      "maxCount": 3,
+      "minCount": 1,
+      "osDiskSizeGB": 32,
+      "vmSize": "Standard_D2s_v3"
+    }
   },
   "svcAcrName": "arohcpsvcint",
   "vnetAddressPrefix": "10.128.0.0/14"

diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json
@@ -136,7 +136,14 @@
       "kvSoftDelete": false
     },
     "rg": "hcp-underlay-usw3tst-svc",
-    "subscription": "ARO Hosted Control Planes (EA Subscription 1)"
+    "subscription": "ARO Hosted Control Planes (EA Subscription 1)",
+    "userAgentPool": {
+      "azCount": 3,
+      "maxCount": 3,
+      "minCount": 1,
+      "osDiskSizeGB": 32,
+      "vmSize": "Standard_D2s_v3"
+    }
   },
   "svcAcrName": "arohcpsvcdev",
   "vnetAddressPrefix": "10.128.0.0/14"

diff --git a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam
@@ -9,6 +9,12 @@ param aksClusterName = '{{ .aksName }}'
 param aksKeyVaultName = '{{ .svc.etcd.kvName }}'
 param aksEtcdKVEnableSoftDelete = {{ .svc.etcd.kvSoftDelete }}
 
+param userAgentMinCount = {{ .svc.userAgentPool.minCount }}
+param userAgentMaxCount = {{ .svc.userAgentPool.maxCount }}
+param userAgentVMSize = '{{ .svc.userAgentPool.vmSize }}'
+param aksUserOsDiskSizeGB = {{ .svc.userAgentPool.osDiskSizeGB }}
+param userAgentPoolAZCount = {{ .svc.userAgentPool.azCount }}
+
 param disableLocalAuth = {{ .frontend.cosmosDB.disableLocalAuth }}
 param deployFrontendCosmos = {{ .frontend.cosmosDB.deploy }}
 param rpCosmosDbName = '{{ .frontend.cosmosDB.name }}'

diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep
@@ -13,6 +13,18 @@ param aksSystemOsDiskSizeGB int = 32
 @description('Disk size for the AKS user nodes')
 param aksUserOsDiskSizeGB int = 32
 
+@description('Min replicas for the worker nodes')
+param userAgentMinCount int = 1
+
+@description('Max replicas for the worker nodes')
+param userAgentMaxCount int = 3
+
+@description('VM instance type for the worker nodes')
+param userAgentVMSize string = 'Standard_D2s_v3'
+
+@description('Availability Zone count for worker nodes')
+param userAgentPoolAZCount int = 3
+
 @description('Names of additional resource group contains ACRs the AKS cluster will get pull permissions on')
 param acrPullResourceGroups array = []
 
@@ -165,6 +177,10 @@ module svcCluster '../modules/aks-cluster-base.bicep' = {
     clusterType: 'svc-cluster'
     systemOsDiskSizeGB: aksSystemOsDiskSizeGB
     userOsDiskSizeGB: aksUserOsDiskSizeGB
+    userAgentMinCount: userAgentMinCount
+    userAgentPoolAZCount: userAgentPoolAZCount
+    userAgentMaxCount: userAgentMaxCount
+    userAgentVMSize: userAgentVMSize
     workloadIdentities: items({
       frontend_wi: {
         uamiName: 'frontend'