ExtraGateway Ingress (#65)

* Create objects to facilitate ingress from an extragateway network

* Add retries to builds since nodes can be in a precarious state during post-deployment

* Increase build retries and add conditional to prevent eg IC being created

* Add quotes to missing build Ansible task name

* Add quotes to missing build Ansible task name

* Alter EG network name var to match deploy vars.yml, and add -ve route selector so eg/net2 routes don't go onto default ic

* Tweak net2 LB pool selection, and add conditional so that internet only clusters don't have ukc-ingress jobs

* Late tweak to ensure EG lb pool selection works

Co-authored-by: Gareth Ellner <29887711+gellner@users.noreply.github.com>

post-deployment/openstack/eg-ingress/build/Dockerfile

@@ -0,0 +1,18 @@
+# FROM ansible-runner:latest
+FROM docker.io/ansible/ansible-runner@sha256:4c6034798b5e724c5a59466f5438e14480c53bc4adac5cc9db5e3997a286a0e4
+
+WORKDIR /usr/local/
+
+ADD ./playbooks /usr/local/playbooks
+COPY ./pip-requirements.txt /usr/local/pip-requirements.txt
+COPY ./ansible-requirements.yml /usr/local/ansible-requirements.yml
+
+# Upgrade pip and install requirements for Kubernetes/Openstack modules
+RUN pip3 install --upgrade pip && \
+    pip3 install -U setuptools && \
+    pip3 install -r /usr/local/pip-requirements.txt --ignore-installed PyYAML
+
+# Install Ansible collections for Kubernetes/Openstack modules
+RUN ansible-galaxy collection install -r /usr/local/ansible-requirements.yml
+
+ENTRYPOINT /usr/local/playbooks/entrypoint.sh

post-deployment/openstack/eg-ingress/build/ansible-requirements.yml

@@ -0,0 +1,6 @@
+---
+collections:
+- name: openstack.cloud
+  version: "1.2.0"
+- name: community.kubernetes
+  version: "1.2.0"

post-deployment/openstack/eg-ingress/build/pip-requirements.txt

@@ -0,0 +1,6 @@
+openshift~=0.12.0
+PyYAML~=5.4.1
+jmespath~=0.10.0
+openstacksdk~=0.56.0
+python-octaviaclient~=2.3.0
+python-openstackclient~=5.5.0

post-deployment/openstack/eg-ingress/build/playbooks/entrypoint.sh

@@ -0,0 +1,10 @@
+export K8S_AUTH_VERIFY_SSL=no
+
+# Add uid to /etc/passwd to allow Ansible to run
+if [ `id -u` -ge 500 ]; then
+    echo "runner:x:`id -u`:`id -g`:,,,:/runner:/bin/bash" > /tmp/passwd
+    cat /tmp/passwd >> /etc/passwd
+    rm /tmp/passwd
+fi
+
+ansible-playbook /usr/local/playbooks/update_eg_loadbalancer_members.yaml

post-deployment/openstack/eg-ingress/build/playbooks/update_eg_loadbalancer_members.yaml

@@ -0,0 +1,95 @@
+---
+- hosts: localhost
+  connection: local
+  gather_facts: no
+
+  tasks:
+  - name: 'Retrieve OpenStack secret'
+    community.kubernetes.k8s_info:
+      kind: Secret
+      name: openstack-credentials
+      namespace: kube-system
+    register: openstackCredentialRaw
+
+  - name: 'Extract OpenStack credential'
+    ansible.builtin.set_fact:
+      openstackCredential: "{{ openstackCredentialRaw | to_json | from_json | json_query(openstackCredentialQuery) }}"
+    vars:
+      openstackCredentialQuery: 'resources[0].data."clouds.yaml"'
+
+  - name: 'Write OpenStack credential to file'
+    ansible.builtin.copy:
+      dest: /etc/openstack/clouds.yaml
+      content: "{{ openstackCredential | b64decode }}"
+
+  - name: 'Get infra nodes'
+    community.kubernetes.k8s_info:
+      kind: Node
+      label_selectors:
+        - node-role.kubernetes.io/infra =
+    register: infraNodes
+
+  - name: 'Extract infra node IPs'
+    ansible.builtin.set_fact:
+      infraMembersRaw: "{{ infraNodes | to_json | from_json | json_query(infraMembersRawQuery) }}"
+    vars:
+      infraMembersRawQuery: "resources[*].status.addresses[?type=='InternalIP'].address"
+
+  - name: 'Build infra node IP dict'
+    ansible.builtin.set_fact:
+      infraMembers: "{{ infraMembers | default([]) + [ item ] }}"
+    with_items: "{{ infraMembersRaw }}"
+
+  - name: 'Get loadbalancer pools'
+    ansible.builtin.shell: openstack loadbalancer pool list -c name -f value | grep -- -eg-
+    register: loadbalancerPoolsRaw
+
+  - name: 'Sanitise loadbalancer pools'
+    ansible.builtin.set_fact:
+      loadbalancerPools: "{{ loadbalancerPoolsRaw.stdout_lines }}"
+
+  - name: 'Get loadbalancer pool members'
+    ansible.builtin.command: openstack loadbalancer member list "{{ loadbalancerPools | first }}" -c address -f value
+    register: loadbalancerMembersRaw
+
+  - name: 'Sanitise loadbalancer pool members'
+    ansible.builtin.set_fact:
+      loadbalancerMembers: "{{ loadbalancerMembersRaw.stdout_lines }}"
+
+  - name: 'Determine members to be added'
+    ansible.builtin.set_fact:
+      loadbalancerMembersToAdd: "{{ loadbalancerMembersToAdd | default([]) + [ item ] }}"
+    with_items: "{{ infraMembers }}"
+    when: 
+      - infraMembers is defined
+      - item not in loadbalancerMembers
+
+  - name: 'Determine members to be deleted'
+    ansible.builtin.set_fact:
+      loadbalancerMembersToDelete: "{{ loadbalancerMembersToDelete | default([]) + [ item ] }}"
+    with_items: "{{ loadbalancerMembers }}"
+    when: 
+      - infraMembers is defined
+      - item not in infraMembers
+
+  - name: 'Add loadbalancer members'
+    openstack.cloud.lb_member:
+      name: "{{ item[1] }}"
+      address: "{{ item[1] }}"
+      pool: "{{ item[0] }}"
+      protocol_port: "{{ 30443 if '-HTTPS-' in item[0] else 30080 }}"
+      state: present
+    with_nested:
+      - "{{ loadbalancerPools }}"
+      - "{{ loadbalancerMembersToAdd }}"
+    when: loadbalancerMembersToAdd is defined
+
+  - name: 'Delete loadbalancer members'
+    openstack.cloud.lb_member:
+      name: "{{ item[1] }}"
+      pool: "{{ item[0] }}"
+      state: absent
+    with_nested:
+      - "{{ loadbalancerPools }}"
+      - "{{ loadbalancerMembersToDelete }}"
+    when: loadbalancerMembersToDelete is defined

post-deployment/openstack/eg-ingress/main.yml

@@ -0,0 +1,131 @@
+---
+- hosts: localhost
+  connection: local
+
+  vars:
+    namespace: "ukc-ingress"
+    extraGatewayNetworkName: "{{ egExternalNetwork | replace(' ','-') | replace('_','-') | lower }}"
+
+  tasks:
+  - name: Create ImageStream
+    k8s:
+      state: present
+      definition:
+        apiVersion: image.openshift.io/v1
+        kind: ImageStream
+        metadata:
+          name: update-eg-loadbalancer-members
+          namespace: "{{ namespace }}"
+        spec:
+          lookupPolicy:
+            local: true
+
+  - name: Create BuildConfig
+    k8s:
+      state: present
+      definition:
+        apiVersion: build.openshift.io/v1
+        kind: BuildConfig
+        metadata:
+          labels:
+            build: update-eg-loadbalancer-members
+          name: update-eg-loadbalancer-members
+          namespace: "{{ namespace }}"
+        spec:
+          output:
+            to:
+              kind: ImageStreamTag
+              name: update-eg-loadbalancer-members:latest
+          source:
+            binary: {}
+            type: Binary
+          strategy:
+            dockerStrategy: {}
+            type: Docker
+
+  - name: Run Docker build
+    command: "oc start-build update-eg-loadbalancer-members --from-dir . -F -n {{ namespace }}"
+    args:
+      chdir: "./build"
+    register: egLoadbalancerBuildStatus
+    until:
+      - "'Push successful' in egLoadbalancerBuildStatus.stdout_lines"
+    retries: 5
+    delay: 60
+
+  - name: Delete BuildConfig
+    k8s:
+      state: absent
+      definition:
+        apiVersion: build.openshift.io/v1
+        kind: BuildConfig
+        metadata:
+          name: update-eg-loadbalancer-members
+          namespace: "{{ namespace }}"
+
+  - name: Create update-eg-loadbalancer-members CronJob
+    k8s:
+      state: present
+      definition: "{{ lookup('template', 'templates/cronjob-update-eg-loadbalancer-members.j2') }}"
+
+  - name: Create IngressController
+    k8s:
+      state: present
+      definition:
+        apiVersion: operator.openshift.io/v1
+        kind: IngressController
+        metadata:
+          namespace: openshift-ingress-operator
+          name: extragateway
+        spec:
+          domain: "{{ extraGatewayNetworkName }}.{{ domainSuffix }}"
+          nodePlacement:
+            nodeSelector:
+              matchLabels:
+                node-role.kubernetes.io/infra: ""
+          endpointPublishingStrategy:
+            type: NodePortService
+          replicas: 2
+          routeSelector:
+            matchExpressions:
+            - key: network
+              operator: In
+              values:
+              - "{{ extraGatewayNetworkName }}"
+
+  - name: Statically set NodePortService port numbers
+    k8s:
+      state: present
+      definition:
+        apiVersion: v1
+        kind: Service
+        metadata:
+          name: router-nodeport-extragateway
+          namespace: openshift-ingress
+        spec:
+          ports:
+          - name: http
+            nodePort: 30080
+            port: 80
+            protocol: TCP
+            targetPort: http
+          - name: https
+            nodePort: 30443
+            port: 443
+            protocol: TCP
+            targetPort: https
+
+  - name: Ensure default IngressController does not pickup EG routes
+    k8s:
+      state: present
+      definition:
+        apiVersion: operator.openshift.io/v1
+        kind: IngressController
+        metadata:
+          name: default
+          namespace: openshift-ingress-operator
+        spec:
+          routeSelector:
+            matchExpressions:
+            - key: network
+              operator: DoesNotExist

post-deployment/openstack/net2-ingress/templates/cronjob-update-loadbalancer-members.j2 → post-deployment/openstack/eg-ingress/templates/cronjob-update-eg-loadbalancer-members.j2

@@ -1,7 +1,7 @@
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
-  name: "update-loadbalancer-members"
+  name: "update-eg-loadbalancer-members"
   namespace: "{{ namespace }}"
 spec:
   schedule: "*/5 * * * *"
@@ -14,8 +14,8 @@ spec:
       template:
         spec:
           containers:
-          - name: update-loadbalancer-members
-            image: update-loadbalancer-members
+          - name: update-eg-loadbalancer-members
+            image: update-eg-loadbalancer-members
             env:
             - name: K8S_AUTH_API_KEY
               valueFrom:

post-deployment/openstack/ingress/build/playbooks/templates/eg_ingress_controller.j2

@@ -0,0 +1,7 @@
+apiVersion: operator.openshift.io/v1
+kind: IngressController
+metadata:
+    namespace: openshift-ingress-operator
+    name: extragateway
+spec:
+    replicas: {{ infraNodes.resources | length }}

post-deployment/openstack/ingress/build/playbooks/update_ingress_controllers.yaml

@@ -18,19 +18,32 @@
         - node-role.kubernetes.io/net2 =
     register: net2Nodes
 
+  - name: 'Get ingresscontrollers'
+    community.kubernetes.k8s_info:
+      kind: IngressController
+      namespace: openshift-ingress-operator
+      name: extragateway
+    register: egIngressController
+
   - name: 'Extract net2 network name'
     ansible.builtin.set_fact:
       net2Networks: "{{ net2Nodes | to_json | from_json | json_query(net2NetworksQuery) }}"
     vars:
       net2NetworksQuery: "resources[*].metadata.labels.network"
 
-  - name: Update default ingresscontroller replicas
-    k8s:
+  - name: 'Update default ingresscontroller replicas'
+    community.kubernetes.k8s:
       state: present
       definition: "{{ lookup('template', 'templates/default_ingress_controller.j2') }}"
 
-  - name: Update net2 ingresscontroller replicas
-    k8s:
+  - name: 'Update net2 ingresscontroller replicas'
+    community.kubernetes.k8s:
       state: present
       definition: "{{ lookup('template', 'templates/net2_ingress_controller.j2') }}"
     with_items: "{{ net2Networks | unique }}"
+
+  - name: 'Update extragateway ingresscontroller replicas'
+    community.kubernetes.k8s:
+      state: present
+      definition: "{{ lookup('template', 'templates/eg_ingress_controller.j2') }}"
+    when: egIngressController.resources | length > 0

post-deployment/openstack/ingress/main.yml

@@ -70,7 +70,10 @@
     args:
       chdir: "./build"
     register: ingressBuildStatus
-    failed_when: "'Push successful' not in ingressBuildStatus.stdout_lines"
+    until:
+      - "'Push successful' in ingressBuildStatus.stdout_lines"
+    retries: 5
+    delay: 60
 
   - name: Delete BuildConfig
     k8s:

post-deployment/openstack/net2-ingress/build/playbooks/entrypoint.sh

@@ -7,4 +7,4 @@ if [ `id -u` -ge 500 ]; then
     rm /tmp/passwd
 fi
 
-ansible-playbook /usr/local/playbooks/update_loadbalancer_members.yaml
+ansible-playbook /usr/local/playbooks/update_net2_loadbalancer_members.yaml

post-deployment/openstack/net2-ingress/build/playbooks/update_loadbalancer_members.yaml → post-deployment/openstack/net2-ingress/build/playbooks/update_net2_loadbalancer_members.yaml

@@ -41,10 +41,10 @@
     with_items: "{{ net2MembersRaw }}"
 
   - name: 'Get loadbalancer pools'
-    ansible.builtin.command: openstack loadbalancer pool list -c name -f value
+    ansible.builtin.shell: openstack loadbalancer pool list -c name -f value | grep -- -net2-
     register: loadbalancerPoolsRaw
 
-  - name: Sanitise loadbalancer pools
+  - name: 'Sanitise loadbalancer pools'
     ansible.builtin.set_fact:
       loadbalancerPools: "{{ loadbalancerPoolsRaw.stdout_lines }}"