Add support for horizontal pod autoscaling

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -1555,10 +1555,144 @@ spec:
                 description: Number of web instance replicas
                 type: integer
                 format: int32
+              web_max_replicas:
+                description: Maximum number of web instances that can be created.
+                type: integer
+                format: int32
               task_replicas:
                 description: Number of task instance replicas
                 type: integer
                 format: int32
+              task_max_replicas:
+                description: Maximum number of task instances that can be created.
+                type: integer
+                format: int32
+              web_hpa_metrics:
+                items:
+                  type: object
+                  properties:
+                    type:
+                      type: string
+                    resource:
+                      properties:
+                        name:
+                          type: string
+                        target:
+                          properties:
+                            type:
+                              type: string
+                            averageUtilization:
+                              type: integer
+                            averageValue:
+                              type: string
+                            value:
+                              type: string
+                          type: object
+                      type: object
+                type: array
+              task_hpa_metrics:
+                items:
+                  type: object
+                  properties:
+                    type:
+                      type: string
+                    resource:
+                      properties:
+                        name:
+                          type: string
+                        target:
+                          properties:
+                            type:
+                              type: string
+                            averageUtilization:
+                              type: integer
+                            averageValue:
+                              type: string
+                            value:
+                              type: string
+                          type: object
+                      type: object
+                type: array
+              web_hpa_behavior:
+                description: Controls the behavior of the HPA when acting on the web pods
+                type: object
+                properties:
+                  scaleUp:
+                    type: object
+                    properties:
+                      selectPolicy:
+                        type: string
+                      stabilizationWindowSeconds:
+                        type: integer
+                      policies:
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            type:
+                              type: string
+                            value:
+                              type: integer
+                            periodSeconds:
+                              type: integer
+                  scaleDown:
+                    type: object
+                    properties:
+                      selectPolicy:
+                        type: string
+                      stabilizationWindowSeconds:
+                        type: integer
+                      policies:
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            type:
+                              type: string
+                            value:
+                              type: integer
+                            periodSeconds:
+                              type: integer
+              task_hpa_behavior:
+                description: Controls the behavior of the HPA when acting on the task pods
+                type: object
+                properties:
+                  scaleUp:
+                    type: object
+                    properties:
+                      selectPolicy:
+                        type: string
+                      stabilizationWindowSeconds:
+                        type: integer
+                      policies:
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            type:
+                              type: string
+                            value:
+                              type: integer
+                            periodSeconds:
+                              type: integer
+                  scaleDown:
+                    type: object
+                    properties:
+                      selectPolicy:
+                        type: string
+                      stabilizationWindowSeconds:
+                        type: integer
+                      policies:
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            type:
+                              type: string
+                            value:
+                              type: integer
+                            periodSeconds:
+                              type: integer
               garbage_collect_secrets:
                 description: Whether or not to remove secrets upon instance removal
                 default: false

config/rbac/role.yaml

@@ -100,6 +100,18 @@ rules:
       - statefulsets/scale
     verbs:
       - patch
+  - apiGroups:
+      - autoscaling
+    resources:
+      - horizontalpodautoscalers
+    verbs:
+      - get
+      - list
+      - create
+      - delete
+      - patch
+      - update
+      - watch
   - apiGroups:
       - ""
     resources:

docs/user-guide/advanced-configuration/horizontal-pod-autoscaler.md

@@ -0,0 +1,63 @@
+### Horizontal Pod Autoscaler (HPA)
+
+The Horizontal Pod Autoscaler feature allows the operator to configure
+HPA resources for the web and task deployments.
+
+The configuration for HPA is cluster and use case specific.  Limits,
+requests, and health probes should be configured for deployments
+associated with a HPA configuration.
+
+**Note:** Any examples provided in this document are for demonstration purposes only
+
+| Name              | Description                                    | Default |
+| ------------------| ---------------------------------------------- | ------- |
+| task_replicas     | Mininum number of replicas of the task pod     | 1       |
+| task_max_replicas | Maximum number of replicas the HPA can create  |         |
+|                   | Set this option to be greater than replicas to |         |
+|                   | activate the HPA feature.                      |         |
+| task_hpa_metrics  | See 'Metrics' section below                    |         |
+| task_hpa_behavior | See 'Behavior' section below                   |         |
+
+| Name              | Description                                    | Default |
+| ------------------| ---------------------------------------------- | ------- |
+| web_replicas      | Mininum number of replicas of the web pod      | 1       |
+| web_max_replicas  | Maximum number of replicas the HPA can create  |         |
+|                   | Set this option to be greater than replicas to |         |
+|                   | activate the HPA feature.                      |         |
+| web_hpa_metrics   | See 'Metrics' section below                    |         |
+| web_hpa_behavior  | See 'Behavior' section below                   |         |
+
+##### Metrics
+
+The metrics option allows configuration of the rules to evaluate when the HPA should
+adjust the number of replicas.  This option currently doesn't support external or
+custom metrics.
+
+Additional information on the options available can be found in the Kubernetes documentation on HPAs.
+
+```yaml
+  spec:
+    task_hpa_metrics:
+      - type: Resource
+        resource:
+          name: memory
+          target:
+            type: Utilization
+            averageUtilization: 90
+```
+
+##### Behavior
+
+The behavior option allows configuration of the rules to modify how quickly or
+slowly the HPA acts on the metrics rules.  Additional information on the options
+available can be found in the Kubernetes documentation on HPAs.
+
+```yaml
+  spec:
+    scaleDown:
+      stabilizationWindowSeconds: 300
+      policies:
+        - type: Percent
+          value: 50
+          periodSeconds: 15
+```

roles/installer/tasks/hpa_configuration.yml

@@ -0,0 +1,16 @@
+---
+- name: Create horizontal pod autoscaler for task pod
+  k8s:
+    apply: yes
+    definition: "{{ lookup('template', 'hpa/task.yaml.j2') }}"
+  when:
+    - task_max_replicas is defined
+    - ( task_max_replicas | int > replicas ) or ( task_max_replicas | int > task_replicas )
+
+- name: Create horizontal pod autoscaler for web pod
+  k8s:
+    apply: yes
+    definition: "{{ lookup('template', 'hpa/web.yaml.j2') }}"
+  when:
+    - web_max_replicas is defined
+    - ( web_max_replicas | int > replicas ) or ( web_max_replicas | int > web_replicas )

roles/installer/tasks/install.yml

@@ -140,6 +140,10 @@
   include_tasks: initialize_django.yml
   when: awx_task_pod_name != ''
 
+- name: Configure hortizonal pod autoscaling (HPA)
+  include_tasks: hpa_configuration.yml
+  when: task_max_replicas is defined or web_max_replicas is defined
+
 - name: Update status variables
   include_tasks: update_status.yml
 

roles/installer/templates/deployments/task.yaml.j2

@@ -8,9 +8,9 @@ metadata:
     {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
-{% if task_replicas != '' %}
+{% if task_replicas != '' and task_max_replicas is not defined %}
   replicas: {{ task_replicas }}
-{% elif replicas != '' %}
+{% elif replicas != '' and task_max_replicas is not defined %}
   replicas: {{ replicas }}
 {% endif %}
   selector:

roles/installer/templates/deployments/web.yaml.j2

@@ -9,9 +9,9 @@ metadata:
     {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
-{% if web_replicas != '' %}
+{% if web_replicas != '' and web_max_replicas is not defined %}
   replicas: {{ web_replicas }}
-{% elif replicas != '' %}
+{% elif replicas != '' and web_max_replicas is not defined %}
   replicas: {{ replicas }}
 {% endif %}
   selector:

roles/installer/templates/hpa/task.yaml.j2

@@ -0,0 +1,28 @@
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: '{{ ansible_operator_meta.name }}-task'
+  namespace: '{{ ansible_operator_meta.namespace }}'
+  labels:
+    {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=4) | trim }}
+    {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
+spec:
+{% if task_replicas != '' and task_replicas | int > 0 %}
+  minReplicas: {{ task_replicas }}
+{% elif replicas != '' and task_replicas | int > 0 %}
+  minReplicas: {{ replicas }}
+{% endif %}
+  maxReplicas: {{ task_max_replicas }}
+  metrics:
+{% if task_hpa_metrics is defined %}
+  metrics:
+    {{ task_hpa_metrics | to_nice_yaml | indent(width=4) }}
+{% endif %}
+{% if task_hpa_behavior is defined %}
+  behavior:
+    {{ task_hpa_behavior | to_nice_yaml | indent(width=4) }}
+{% endif %}
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: '{{ ansible_operator_meta.name }}-task'

roles/installer/templates/hpa/web.yaml.j2

@@ -0,0 +1,28 @@
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: '{{ ansible_operator_meta.name }}-web'
+  namespace: '{{ ansible_operator_meta.namespace }}'
+  labels:
+    {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=4) | trim }}
+    {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
+spec:
+{% if web_replicas != '' and web_replicas | int > 0 %}
+  minReplicas: {{ web_replicas }}
+{% elif replicas != '' and web_replicas | int > 0 %}
+  minReplicas: {{ replicas }}
+{% endif %}
+  maxReplicas: {{ web_max_replicas }}
+  metrics:
+{% if web_hpa_metrics is defined %}
+  metrics:
+    {{ web_hpa_metrics | to_nice_yaml | indent(width=4) }}
+{% endif %}
+{% if web_hpa_behavior is defined %}
+  behavior:
+    {{ web_hpa_behavior | to_nice_yaml | indent(width=4) }}
+{% endif %}
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: '{{ ansible_operator_meta.name }}-web'