[Release-1.29] - Allow setting priorityClassName on ServiceLB daemonset.

[brandond]

Backport fix for Allow setting priorityClassName on ServiceLB daemonset.

• Allow setting priorityClassName on ServiceLB daemonset. #10033

[mdrahman-suse]

Validated with release-1.29 branch on commit 12864fb

Testing

• Steps: Allow setting priorityClassName on ServiceLB daemonset. #10033 (comment)

Validation

$ k3s -v
k3s version v1.29.5+k3s-12864fb6 (12864fb6)
go version go1.21.9

$ kubectl apply -f pc.yaml
priorityclass.scheduling.k8s.io/high-priority created

$ kubectl get priorityClass
NAME                      VALUE        GLOBAL-DEFAULT   AGE
high-priority             1000000      false            5m30s
system-cluster-critical   2000000000   false            6m53s
system-node-critical      2000001000   false            6m53s

$ kubectl get pods -A
NAMESPACE     NAME                                     READY   STATUS      RESTARTS   AGE
kube-system   coredns-6799fbcd5-rbwx9                  1/1     Running     0          91s
kube-system   helm-install-traefik-4jc8z               0/1     Completed   1          92s
kube-system   helm-install-traefik-crd-ztl6n           0/1     Completed   0          92s
kube-system   local-path-provisioner-6f5d79df6-6vswn   1/1     Running     0          91s
kube-system   metrics-server-54fd9b65b-2hcrf           1/1     Running     0          91s
kube-system   svclb-traefik-c02c1f5e-tzvj7             2/2     Running     0          75s
kube-system   traefik-7d5f6474df-87n6c                 1/1     Running     0          75s

$ kubectl get pods -n kube-system svclb-traefik-c02c1f5e-tzvj7 -o yaml | grep priority
  priority: 2000001000
  priorityClassName: system-node-critical

$ kubectl annotate svc -n kube-system traefik svccontroller.k3s.cattle.io/priorityclassname=high-priority
service/traefik annotated

$ kubectl get svc -n kube-system traefik -o  yaml | grep priority
    svccontroller.k3s.cattle.io/priorityclassname: high-priority

$ kubectl get pods -A
NAMESPACE     NAME                                     READY   STATUS      RESTARTS   AGE
kube-system   coredns-6799fbcd5-rbwx9                  1/1     Running     0          3m43s
kube-system   helm-install-traefik-4jc8z               0/1     Completed   1          3m44s
kube-system   helm-install-traefik-crd-ztl6n           0/1     Completed   0          3m44s
kube-system   local-path-provisioner-6f5d79df6-6vswn   1/1     Running     0          3m43s
kube-system   metrics-server-54fd9b65b-2hcrf           1/1     Running     0          3m43s
kube-system   svclb-traefik-c02c1f5e-wq2rm             2/2     Running     0          18s
kube-system   traefik-7d5f6474df-87n6c                 1/1     Running     0          3m27s

$ kubectl get pods -n kube-system svclb-traefik-c02c1f5e-wq2rm -o yaml | grep priority
  priority: 1000000
  priorityClassName: high-priority