Merge branch 'main' into T243200

.gitattributes

@@ -1,7 +1,10 @@
 # Directory entries are not enough to encrypt fines beneath it
 # https://github.com/AGWA/git-crypt#gitattributes-file
 paws/secrets.yaml filter=git-crypt diff=git-crypt
-cloud.conf filter=git-crypt diff=git-crypt
+ansible/files/csi-secret-cinderplugin.yaml.eqiad1.crypt filter=git-crypt diff=git-crypt
+ansible/files/csi-secret-cinderplugin.yaml.codfw1dev.crypt filter=git-crypt diff=git-crypt
 paws/codfw-secrets.yaml filter=git-crypt diff=git-crypt
 paws/files/minesweeper/secrets/** filter=git-crypt diff=git-crypt
-terraform/secrets.tf filter=git-crypt diff=git-crypt
+tofu/secrets.tf filter=git-crypt diff=git-crypt
+secrets-eqiad1.sh filter=git-crypt diff=git-crypt
+secrets-codfw1dev.sh filter=git-crypt diff=git-crypt

.github/workflows/jobber.yaml

@@ -10,7 +10,7 @@ name: build and push jobber
 jobs:
   build-and-push:
     name: build and push jobber
-    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v2
+    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v3
     with:
       imagename: jobber
     secrets:

.github/workflows/minesweeper.yaml

@@ -10,7 +10,7 @@ name: build and push minesweeper
 jobs:
   build-and-push:
     name: build and push minesweeper
-    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v2
+    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v3
     with:
       imagename: minesweeper
     secrets:

.github/workflows/nbserve.yaml

@@ -10,7 +10,7 @@ name: build and push nbserve
 jobs:
   build-and-push:
     name: build and push nbserve
-    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v2
+    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v3
     with:
       imagename: nbserve
     secrets:

.github/workflows/paws-hub.yaml

@@ -10,7 +10,7 @@ name: build and push paws-hub
 jobs:
   build-and-push:
     name: build and push paws-hub
-    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v2
+    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v3
     with:
       imagename: paws-hub
     secrets:

.github/workflows/renderer.yaml

@@ -10,7 +10,7 @@ name: build and push renderer
 jobs:
   build-and-push:
     name: build and push renderer
-    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v2
+    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v3
     with:
       imagename: renderer
     secrets:

.github/workflows/singleuser.yaml

@@ -11,7 +11,7 @@ name: build and push singleuser
 jobs:
   build-and-push:
     name: build and push singleuser
-    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v2
+    uses: toolforge/github-actions/.github/workflows/build-and-push.yaml@build-and-push-v3
     with:
       imagename: singleuser
     secrets:

.github/workflows/tofu-fmt.yaml

@@ -0,0 +1,28 @@
+---
+
+name: "tofu fmt"
+
+'on':
+  pull_request:
+
+jobs:
+  tf-fmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: install tofu
+        run: |
+          sudo apt-get update && sudo apt-get install -y gnupg software-properties-common
+          curl -s https://packagecloud.io/install/repositories/opentofu/tofu/script.deb.sh?any=true -o /tmp/tofu-repository-setup.sh
+          sudo bash /tmp/tofu-repository-setup.sh
+          sudo apt install tofu
+      - name: tofu fmt
+        run: |
+          shopt -s extglob
+          cd tofu
+          if ! tofu fmt -check -diff !(secrets).tf ; then
+             echo "please update your tofu code to match the above.";
+             echo 'or run `tofu fmt` to have tofu reformat it.';
+             exit 1;
+          fi

.gitignore

@@ -10,3 +10,6 @@ dev-values.yaml
 *.swp
 terraform.tfstate*
 .terraform*
+tofu/kube.config
+.venv*
+ansible/collections/*

README.md

@@ -24,8 +24,8 @@ If your PR originates from a fork, please be sure "Allow edits and access to sec
 It is possible to run a fully-functioning PAWS system inside [minikube](https://minikube.sigs.k8s.io/docs/)! You don't need
 access to the secrets.yaml file to do it either, since the defaults mostly support it.
 
-You will need to install minikube (tested on minikube 1.23) and [helm](https://helm.sh) and kubectl on your system. When you are confident those are working, start minikube with:
- - `minikube start --kubernetes-version=v1.23.15`
+You will need to install minikube (tested on minikube v1.33.1) and [helm](https://helm.sh) and kubectl on your system. When you are confident those are working, start minikube with:
+ - `minikube start --kubernetes-version=v1.26.8`
  - `minikube addons enable ingress`
 (from the top level of this repo):
 install the dependencies for the PAWS dev environment with these steps:
@@ -110,55 +110,17 @@ Bug: T318182
 
 ### Deployment ###
 ```
-cd terraform
-terraform apply -var datacenter=<eqiad1|codfw1dev>
-
-mkdir /tmp/paws-k8s-setup/
-git clone https://github.com/kubernetes/cloud-provider-openstack.git /tmp/paws-k8s-setup/cloud-provider-openstack/
-cp cloud.conf /tmp/paws-k8s-setup/
-cd /tmp/paws-k8s-setup/cloud-provider-openstack/
-git checkout 9ed6d961c6ee5a4f51533877ae981aa6d9753f2d # newest has so far worked though
-base64 -w 0 ../cloud.conf ; echo
-vim manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml # replace cloud.conf 64 with above
-kubectl create -f manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml
-kubectl -f manifests/cinder-csi-plugin/ apply
-cd -
-```
-
-sc.yaml:
-
-```
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: standard
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-provisioner: cinder.csi.openstack.org
-parameters:
-  availability: nova
-```
-
-```
-kubectl apply -f sc.yaml
-
-helm upgrade --install ingress-nginx ingress-nginx \
-  --version v4.4.0 \
-  --repo https://kubernetes.github.io/ingress-nginx \
-  --namespace ingress-nginx --create-namespace \
-  --set controller.service.type=NodePort \
-  --set controller.service.enableHttps=false \
-  --set controller.service.nodePorts.http=30001 \
-  --set-string controller.config.proxy-body-size="4m" # T328168
-
-kubectl config set-context --current --namespace=prod
-helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/
-helm dep up paws/
-kubectl create namespace prod
-helm install paws --namespace prod ./paws -f paws/secrets.yaml -f paws/production.yaml --timeout=50m
-kubectl apply -f manifests/psp.yaml
+bash deploy.sh <eqiad1|codfw1dev>
 ```
 
 update the web proxy in horizon to point to current cluster.
 
 https://wikitech.wikimedia.org/wiki/PAWS/Admin#Deployment
+
+#### Disaster Recovery
+If the entire project is removed two parts of paws are not managed by tofu/ansible.
+Object storage container: An object storage container named "tofu-state" will need to be generated in horizon. This is where the state file for tofu resides.
+NFS: The NFS server is not included. And a fresh NFS server will be needed for paws to operate.
+
+# backup prometheus
+see ansible/files/prometheus-data.sh for example of backup/restore

ansible/ansible.cfg

@@ -1,5 +1,13 @@
 [defaults]
-
 # Better error output
 stdout_callback=debug
 stderr_callback=debug
+
+collections_path=./collections/ansible_collections
+
+# we're only using localhost, no need for the warning.
+localhost_warning=False
+
+[inventory]
+# Only using localhost, so no inventory
+inventory_unparsed_warning=False

ansible/files/sc.yaml

@@ -0,0 +1,11 @@
+---
+
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: standard
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: cinder.csi.openstack.org
+parameters:
+  availability: nova

ansible/paws.yaml

@@ -5,13 +5,57 @@
   hosts: localhost
   tasks:
     - name: include env vars
-      include_vars: vars/{{ datacenter | default('eqiad1') }}.yaml
+      include_vars: vars/{{ datacenter }}.yaml
+
+    - name: Temporary directory for cloud provider
+      ansible.builtin.tempfile:
+        state: directory
+        suffix: paws
+      register: paws_dir
+      changed_when: False
+
+    - name: Clone cloud-provider-openstack
+      ansible.builtin.git:
+        repo: https://github.com/kubernetes/cloud-provider-openstack.git
+        dest: "{{ paws_dir.path }}"
+        version: v1.26.3
+      changed_when: False
+
+    - name: Copy file with owner and permissions
+      ansible.builtin.copy:
+        src: files/csi-secret-cinderplugin.yaml.{{ datacenter }}.crypt
+        dest: "{{ paws_dir.path }}/manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml"
+      changed_when: False
+
+    - name: install the things!
+      kubernetes.core.k8s:
+        state: present
+        src: "{{ paws_dir.path }}/manifests/cinder-csi-plugin/{{ item }}"
+      loop:
+        - "csi-secret-cinderplugin.yaml"
+        - "cinder-csi-controllerplugin-rbac.yaml"
+        - "cinder-csi-controllerplugin.yaml"
+        - "cinder-csi-nodeplugin-rbac.yaml"
+        - "cinder-csi-nodeplugin.yaml"
+        - "csi-cinder-driver.yaml"
+
+    - name: Delete temporary directory
+      ansible.builtin.file:
+        state: absent
+        path: "{{ paws_dir.path }}"
+      changed_when: False
+
+    - name: and sc.yaml
+      kubernetes.core.k8s:
+        state: present
+        src: "files/sc.yaml"
 
     - name: Deploy ingress-nginx
       kubernetes.core.helm:
         name: ingress-nginx
         chart_ref: ingress-nginx
         chart_repo_url: https://kubernetes.github.io/ingress-nginx
+        chart_version: 4.10.1
         release_namespace: ingress-nginx
         create_namespace: true
         set_values:
@@ -23,6 +67,8 @@
             value_type: string
           - value: controller.config.proxy-body-size=4m
             value_type: string
+          - value: controller.config.allow-snippet-annotations=true
+            value_type: string
 
     - name: Add jupyterhub chart repo
       kubernetes.core.helm_repository:
@@ -36,7 +82,31 @@
         release_namespace: "{{ namespace }}"
         create_namespace: true
         values_files:
-          - ../paws/{{ secret-file }}.yaml
-          - ../paws/{{ env-file }}.yaml
+          - ../paws/{{ secret_file }}.yaml
+          - ../paws/{{ env_file }}.yaml
         timeout: '50m'
         dependency_update: true
+
+    - name: Create metrics namespace for prometheus
+      kubernetes.core.k8s:
+        name: metrics
+        kind: Namespace
+        state: present
+
+    - name: Prometheus
+      kubernetes.core.helm:
+        name: prometheus
+        chart_ref: prometheus
+        chart_repo_url: https://prometheus-community.github.io/helm-charts
+        chart_version: 25.26.0
+        release_namespace: metrics
+        create_namespace: true
+        set_values:
+          - value: prometheus.retention=30d
+            value_type: string
+
+    - name: Ingress for prometheus
+      kubernetes.core.k8s:
+        state: present
+        template: "templates/prometheus-ingress.yaml.j2"
+        namespace: metrics

ansible/templates/prometheus-ingress.yaml.j2

@@ -0,0 +1,21 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: prometheus
+spec:
+  ingressClassName: nginx
+  rules:
+{% if datacenter == 'codfw1dev' %}
+  - host: prometheus-paws.codfw1dev.wmcloud.org
+{% else %}
+  - host: prometheus-paws.wmcloud.org
+{% endif %}
+    http:
+      paths:
+      - backend:
+          service:
+            name: prometheus-server
+            port:
+              number: 80
+        path: /
+        pathType: Prefix