diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..9f401f87 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +--- +name: CI/CD Workflow + +on: + push: + branches: + - main + paths: + - '**.yaml' + - '**/Dockerfile' + - '**/charts/**' + tags: + - 'v*' + +env: + IMAGE_NGINX_CACHE: nginx-cache + REGISTRY: ghcr.io/${{ github.repository_owner }} + PUSH: 1 + LOAD: 1 + NGINX_CACHE_TAG: v0.1.0 + TAG: v0.3.1 + PLATFORM_ARCH: linux/amd64 + +jobs: + build-and-push: + name: Build Cozystack + runs-on: ubuntu-latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 + steps: + - name: Set up Docker Registry + run: | + if [ "$GITHUB_ACTIONS" = "true" ]; then + echo "REGISTRY=ghcr.io/${{ github.repository_owner }}" >> $GITHUB_ENV + else + echo "REGISTRY=localhost:5000/cozystack_local" >> $GITHUB_ENV + fi + + - uses: actions/checkout@v3 + - name: Build usig make + run: | + make + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 00000000..6f8c2b2c --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,22 @@ +name: Run E2E Tests + +on: + pull_request: + branches: + - main + +jobs: + e2e-tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up SSH + uses: webfactory/ssh-agent@v0.5.3 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Run E2E Tests on Remote Server + run: ssh -p 2222 root@mgr.cp.if.ua 'bash -s' < /home/cozystack/hack/e2e.sh diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..a79650b4 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,48 @@ +name: Lint + +on: + push: + branches: [ main ] # Lint only on pushes to the main branch + pull_request: + branches: [ main ] # Lint on PRs targeting the main branch + +permissions: + contents: read + +jobs: + lint: + name: Super-Linter + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Run Super-Linter + uses: github/super-linter@v4 + env: + # To report GitHub Actions status checks + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VALIDATE_ALL_CODEBASE: false # Lint only changed files + VALIDATE_TERRAFORM: false # Disable Terraform linting (remove if you need it) + DEFAULT_BRANCH: main # Set your default branch + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Enable only the linters you need for your project + VALIDATE_JAVASCRIPT_ES: true + VALIDATE_PYTHON_BLACK: true + VALIDATE_HTML: false + VALIDATE_GO: false + VALIDATE_XML: false + VALIDATE_JAVA: false + VALIDATE_DOCKERFILE: false + # turn off JSCPD copy/paste detection, which results in lots of results for examples and devops repos + VALIDATE_JSCPD: false + # turn off shfmt shell formatter as we already have shellcheck + VALIDATE_SHELL_SHFMT: false + VALIDATE_EDITORCONFIG: false + # prevent Kubernetes CRD API's from causing kubeval to fail + # also change schema location to an up-to-date list + # https://github.com/yannh/kubernetes-json-schema/#kubeval + KUBERNETES_KUBEVAL_OPTIONS: --ignore-missing-schemas --schema-location https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/ diff --git a/.github/workflows/linters/.markdown-lint.yml b/.github/workflows/linters/.markdown-lint.yml new file mode 100644 index 00000000..335724b4 --- /dev/null +++ b/.github/workflows/linters/.markdown-lint.yml @@ -0,0 +1,7 @@ +--- + # MD013/line-length - Line length + MD013: + # Number of characters, default is 80 + line_length: 9999 + # check code blocks? + code_blocks: false diff --git a/.github/workflows/linters/.yaml-lint.yml b/.github/workflows/linters/.yaml-lint.yml new file mode 100644 index 00000000..95afab76 --- /dev/null +++ b/.github/workflows/linters/.yaml-lint.yml @@ -0,0 +1,55 @@ + +yaml-files: +- '*.yaml' +- '*.yml' +- '.yamllint' + +rules: + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable + line-length: + max: 130 + allow-non-breakable-words: true + allow-non-breakable-inline-mappings: false diff --git a/packages/apps/ferretdb/ferretdb.svg b/packages/apps/ferretdb/ferretdb.svg new file mode 100644 index 00000000..196871e1 --- /dev/null +++ b/packages/apps/ferretdb/ferretdb.svg @@ -0,0 +1,54 @@ + + + + + + + + + + + + diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index 86b6633b..2c88a526 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/aenix-io/cozystack/nginx-cache:v0.1.0 +mgr.cp.if.ua/nginx-cache:v0.1.0 diff --git a/packages/apps/kubernetes-proxmox/.helmignore b/packages/apps/kubernetes-proxmox/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packages/apps/kubernetes-proxmox/Chart.yaml b/packages/apps/kubernetes-proxmox/Chart.yaml new file mode 100644 index 00000000..ec1ef8f9 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/Chart.yaml @@ -0,0 +1,25 @@ +apiVersion: v2 +name: kubernetes-proxmox +description: Managed Kubernetes service +icon: https://upload.wikimedia.org/wikipedia/commons/thumb/3/39/Kubernetes_logo_without_workmark.svg/723px-Kubernetes_logo_without_workmark.svg.png + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.2.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.19.0" diff --git a/packages/apps/kubernetes-proxmox/Makefile b/packages/apps/kubernetes-proxmox/Makefile new file mode 100644 index 00000000..ecefd153 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/Makefile @@ -0,0 +1,17 @@ +UBUNTU_CONTAINER_DISK_TAG = v1.29.1 + +include ../../../scripts/common-envs.mk + +image: image-ubuntu-container-disk + +image-ubuntu-container-disk: + docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 images/ubuntu-container-disk \ + --provenance false \ + --tag $(REGISTRY)/ubuntu-container-disk:$(call settag,$(UBUNTU_CONTAINER_DISK_TAG)) \ + --tag $(REGISTRY)/ubuntu-container-disk:$(call settag,$(UBUNTU_CONTAINER_DISK_TAG)-$(TAG)) \ + --cache-from type=registry,ref=$(REGISTRY)/ubuntu-container-disk:latest \ + --cache-to type=inline \ + --metadata-file images/ubuntu-container-disk.json \ + --push=$(PUSH) \ + --load=$(LOAD) + echo "$(REGISTRY)/ubuntu-container-disk:$(call settag,$(UBUNTU_CONTAINER_DISK_TAG))" > images/ubuntu-container-disk.tag diff --git a/packages/apps/kubernetes-proxmox/README.md b/packages/apps/kubernetes-proxmox/README.md new file mode 100644 index 00000000..aaa8d398 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/README.md @@ -0,0 +1,28 @@ +# Managed Kubernetes Service + +## Overview + +The Managed Kubernetes Service offers a streamlined solution for efficiently managing server workloads. Kubernetes has emerged as the industry standard, providing a unified and accessible API, primarily utilizing YAML for configuration. This means that teams can easily understand and work with Kubernetes, streamlining infrastructure management. + +The Kubernetes leverages robust software design patterns, enabling continuous recovery in any scenario through the reconciliation method. Additionally, it ensures seamless scaling across a multitude of servers, addressing the challenges posed by complex and outdated APIs found in traditional virtualization platforms. This managed service eliminates the need for developing custom solutions or modifying source code, saving valuable time and effort. + +## Deployment Details + +The managed Kubernetes service deploys a standard Kubernetes cluster utilizing the Cluster API, Kamaji as control-plane provicer and the KubeVirt infrastructure provider. This ensures a consistent and reliable setup for workloads. + +Within this cluster, users can take advantage of LoadBalancer services and easily provision physical volumes as needed. The control-plane operates within containers, while the worker nodes are deployed as virtual machines, all seamlessly managed by the application. + +- Docs: https://github.com/clastix/kamaji +- Docs: https://cluster-api.sigs.k8s.io/ +- GitHub: https://github.com/clastix/kamaji +- GitHub: https://github.com/kubernetes-sigs/cluster-api-provider-kubevirt +- GitHub: https://github.com/kubevirt/csi-driver + + +## How-Tos + +How to access to deployed cluster: + +``` +kubectl get secret -n kubernetes--admin-kubeconfig -o go-template='{{ printf "%s\n" (index .data "super-admin.conf" | base64decode) }}' > test +``` diff --git a/packages/apps/kubernetes-proxmox/images/ubuntu-container-disk.json b/packages/apps/kubernetes-proxmox/images/ubuntu-container-disk.json new file mode 100644 index 00000000..82131b31 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/images/ubuntu-container-disk.json @@ -0,0 +1,4 @@ +{ + "containerimage.config.digest": "sha256:62baab666445d76498fb14cc1d0865fc82e4bdd5cb1d7ba80475dc5024184622", + "containerimage.digest": "sha256:9363d717f966f4e7927da332eaaf17401b42203a2fcb493b428f94d096dae3a5" +} \ No newline at end of file diff --git a/packages/apps/kubernetes-proxmox/images/ubuntu-container-disk.tag b/packages/apps/kubernetes-proxmox/images/ubuntu-container-disk.tag new file mode 100644 index 00000000..c2389748 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/images/ubuntu-container-disk.tag @@ -0,0 +1 @@ +ghcr.io/aenix-io/cozystack/ubuntu-container-disk:v1.29.1 diff --git a/packages/apps/kubernetes-proxmox/images/ubuntu-container-disk/Dockerfile b/packages/apps/kubernetes-proxmox/images/ubuntu-container-disk/Dockerfile new file mode 100644 index 00000000..a30d70f7 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/images/ubuntu-container-disk/Dockerfile @@ -0,0 +1,51 @@ +FROM ubuntu:22.04 as guestfish + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get -y install \ + libguestfs-tools \ + linux-image-generic \ + make \ + bash-completion \ + && apt-get clean + +WORKDIR /build + +FROM guestfish as builder + +RUN wget -O image.img https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img + +RUN qemu-img resize image.img 5G \ + && eval "$(guestfish --listen --network)" \ + && guestfish --remote add-drive image.img \ + && guestfish --remote run \ + && guestfish --remote mount /dev/sda1 / \ + && guestfish --remote command "growpart /dev/sda 1 --verbose" \ + && guestfish --remote command "resize2fs /dev/sda1" \ +# docker repo + && guestfish --remote sh "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg" \ + && guestfish --remote sh 'echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list' \ +# kubernetes repo + && guestfish --remote sh "curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg" \ + && guestfish --remote sh "echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list" \ +# install containerd + && guestfish --remote command "apt-get update -y" \ + && guestfish --remote command "apt-get install -y containerd.io" \ +# configure containerd + && guestfish --remote command "mkdir -p /etc/containerd" \ + && guestfish --remote sh "containerd config default | tee /etc/containerd/config.toml" \ + && guestfish --remote command "sed -i '/SystemdCgroup/ s/=.*/= true/' /etc/containerd/config.toml" \ +# install kubernetes + && guestfish --remote command "apt-get install -y kubelet kubeadm" \ +# clean apt cache + && guestfish --remote sh 'apt-get clean && rm -rf /var/lib/apt/lists/*' \ +# write system configuration + && guestfish --remote sh 'printf "%s\n" net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1 net.ipv6.conf.all.disable_ipv6=0 net.ipv4.tcp_congestion_control=bbr vm.overcommit_memory=1 kernel.panic=10 kernel.panic_on_oops=1 fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=524288 | tee > /etc/sysctl.d/kubernetes.conf' \ + && guestfish --remote sh 'printf "%s\n" overlay br_netfilter | tee /etc/modules-load.d/kubernetes.conf' \ + && guestfish --remote sh "rm -f /etc/resolv.conf && ln -s ../run/systemd/resolve/stub-resolv.conf /etc/resolv.conf" \ +# umount all and exit + && guestfish --remote umount-all \ + && guestfish --remote exit + +FROM scratch +COPY --from=builder /build/image.img /disk/image.qcow2 diff --git a/packages/apps/kubernetes-proxmox/templates/NOTES.txt b/packages/apps/kubernetes-proxmox/templates/NOTES.txt new file mode 100644 index 00000000..23e07dab --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/NOTES.txt @@ -0,0 +1,3 @@ +To get kubeconfig for this cluster run: + +kubectl get secret -n {{ .Release.Namespace }} {{ .Release.Name }}-admin-kubeconfig -o go-template='{{`{{ printf "%s\n" (index .data "super-admin.conf" | base64decode) }}`}}' diff --git a/packages/apps/kubernetes-proxmox/templates/_helpers.tpl b/packages/apps/kubernetes-proxmox/templates/_helpers.tpl new file mode 100644 index 00000000..36c06b64 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "kubernetes.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kubernetes.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kubernetes.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "kubernetes.labels" -}} +helm.sh/chart: {{ include "kubernetes.chart" . }} +{{ include "kubernetes.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kubernetes.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kubernetes.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/packages/apps/kubernetes-proxmox/templates/cloud-config.yaml b/packages/apps/kubernetes-proxmox/templates/cloud-config.yaml new file mode 100644 index 00000000..9f16548a --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/cloud-config.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-cloud-config +data: + cloud-config: | + loadBalancer: + creationPollInterval: 5 + creationPollTimeout: 60 + namespace: {{ .Release.Namespace }} diff --git a/packages/apps/kubernetes-proxmox/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes-proxmox/templates/cluster-autoscaler/deployment.yaml new file mode 100644 index 00000000..ceccf990 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/cluster-autoscaler/deployment.yaml @@ -0,0 +1,86 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-cluster-autoscaler + labels: + app: {{ .Release.Name }}-cluster-autoscaler +spec: + selector: + matchLabels: + app: {{ .Release.Name }}-cluster-autoscaler + replicas: 1 + template: + metadata: + labels: + app: {{ .Release.Name }}-cluster-autoscaler + spec: + containers: + - image: ghcr.io/kvaps/test:cluster-autoscaller + name: cluster-autoscaler + command: + - /cluster-autoscaler + args: + - --cloud-provider=clusterapi + - --kubeconfig=/etc/kubernetes/kubeconfig/super-admin.svc + - --clusterapi-cloud-config-authoritative + - --node-group-auto-discovery=clusterapi:namespace={{ .Release.Namespace }},clusterName={{ .Release.Name }} + volumeMounts: + - mountPath: /etc/kubernetes/kubeconfig + name: kubeconfig + readOnly: true + volumes: + - configMap: + name: {{ .Release.Name }}-cloud-config + name: cloud-config + - secret: + secretName: {{ .Release.Name }}-admin-kubeconfig + name: kubeconfig + serviceAccountName: {{ .Release.Name }}-cluster-autoscaler + terminationGracePeriodSeconds: 10 +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-cluster-autoscaler +subjects: +- kind: ServiceAccount + name: {{ .Release.Name }}-cluster-autoscaler + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-cluster-autoscaler +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-cluster-autoscaler +rules: + - apiGroups: + - cluster.x-k8s.io + resources: + - machinedeployments + - machinedeployments/scale + - machines + - machinesets + - machinepools + verbs: + - get + - list + - update + - watch + - apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - proxmoxmachinetemplates + verbs: + - get + - list + - update + - watch diff --git a/packages/apps/kubernetes-proxmox/templates/cluster.yaml b/packages/apps/kubernetes-proxmox/templates/cluster.yaml new file mode 100644 index 00000000..186ed08f --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/cluster.yaml @@ -0,0 +1,147 @@ +{{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }} +{{- $etcd := index $myNS.metadata.annotations "namespace.cozystack.io/etcd" }} +{{- $ingress := index $myNS.metadata.annotations "namespace.cozystack.io/ingress" }} +{{- $host := index $myNS.metadata.annotations "namespace.cozystack.io/host" }} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}' +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.243.0.0/16 + controlPlaneRef: + namespace: {{ .Release.Namespace }} + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: KamajiControlPlane + name: {{ .Release.Name }} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxCluster + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 +kind: KamajiControlPlane +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + cluster.x-k8s.io/role: control-plane + annotations: + kamaji.clastix.io/kubeconfig-secret-key: "super-admin.svc" +spec: + dataStoreName: "{{ $etcd }}" + addons: + coreDNS: {} + konnectivity: {} + kubelet: + cgroupfs: systemd + preferredAddressTypes: + - InternalIP + - ExternalIP + network: + serviceType: ClusterIP + ingress: + extraAnnotations: + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + hostname: {{ .Values.host | default (printf "%s.%s" .Release.Name $host) }}:443 + className: "{{ $ingress }}" + deployment: + replicas: 2 + version: 1.29.0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: ProxmoxCluster +metadata: + annotations: + cluster.x-k8s.io/managed-by: kamaji + cluster.x-k8s.io/cluster-name: {{ .Release.Name }} + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + spec: + controlPlaneEndpoint: + host: {{ .Values.host | default (printf "%s.%s" .Release.Name $host) }} + port: 443 + ipv4Config: + addresses: ${NODE_IP_RANGES} + prefix: ${IP_PREFIX} + gateway: ${GATEWAY} + dnsServers: ${DNS_SERVERS} + allowedNodes: ${ALLOWED_NODES:=[]} + +{{- range $groupName, $group := .Values.nodeGroups }} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: {{ $.Release.Name }}-{{ $groupName }} + namespace: {{ $.Release.Namespace }} +spec: + template: + spec: + users: + - name: root + sshAuthorizedKeys: [${VM_SSH_KEYS}] + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + provider-id: "proxmox://'{{ ds.meta_data.instance_id }}'" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: ProxmoxMachineTemplate +metadata: + name: {{ $.Release.Name }}-{{ $groupName }} + namespace: {{ $.Release.Namespace }} +spec: + template: + spec: + sourceNode: "${PROXMOX_SOURCENODE}" + templateID: ${TEMPLATE_VMID} + format: "qcow2" + full: true + numSockets: ${NUM_SOCKETS:=2} + numCores: ${NUM_CORES:=4} + memoryMiB: ${MEMORY_MIB:=16384} + disks: + bootVolume: + disk: ${BOOT_VOLUME_DEVICE} + sizeGb: ${BOOT_VOLUME_SIZE:=100} + network: + default: + bridge: ${BRIDGE} + model: virtio +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: {{ $.Release.Name }}-{{ $groupName }} + namespace: {{ $.Release.Namespace }} + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "{{ $group.minReplicas }}" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "{{ $group.maxReplicas }}" + capacity.cluster-autoscaler.kubernetes.io/memory: "{{ $group.resources.memory }}" + capacity.cluster-autoscaler.kubernetes.io/cpu: "{{ $group.resources.cpu }}" +spec: + clusterName: {{ $.Release.Name }} + template: + spec: + clusterName: {{ $.Release.Name }} + version: v1.29.0 + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: {{ $.Release.Name }}-{{ $groupName }} + namespace: default + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxMachineTemplate + name: {{ $.Release.Name }}-{{ $groupName }} + namespace: default +{{- end }} diff --git a/packages/apps/kubernetes-proxmox/templates/clusterctl.yaml b/packages/apps/kubernetes-proxmox/templates/clusterctl.yaml new file mode 100644 index 00000000..531d452e --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/clusterctl.yaml @@ -0,0 +1,29 @@ +## -- Controller settings -- ## +PROXMOX_URL: "https://pve.example:8006" # The Proxmox VE host +PROXMOX_TOKEN: "root@pam!capi" # The Proxmox VE TokenID for authentication +PROXMOX_SECRET: "REDACTED" # The secret associated with the TokenID + + +## -- Required workload cluster default settings -- ## +PROXMOX_SOURCENODE: "pve" # The node that hosts the VM template to be used to provision VMs +TEMPLATE_VMID: "100" # The template VM ID used for cloning VMs +ALLOWED_NODES: "[pve1,pve2,pve3, ...]" # The Proxmox VE nodes used for VM deployments +VM_SSH_KEYS: "ssh-ed25519 ..., ssh-ed25519 ..." # The ssh authorized keys used to ssh to the machines. + +## -- networking configuration-- ## +CONTROL_PLANE_ENDPOINT_IP: "10.10.10.4" # The IP that kube-vip is going to use as a control plane endpoint +NODE_IP_RANGES: "[10.10.10.5-10.10.10.50, ...]" # The IP ranges for Cluster nodes +GATEWAY: "10.10.10.1" # The gateway for the machines network-config. +IP_PREFIX: "25" # Subnet Mask in CIDR notation for your node IP ranges +DNS_SERVERS: "[8.8.8.8,8.8.4.4]" # The dns nameservers for the machines network-config. +BRIDGE: "vmbr1" # The network bridge device for Proxmox VE VMs + +## -- xl nodes -- ## +BOOT_VOLUME_DEVICE: "scsi0" # The device used for the boot disk. +BOOT_VOLUME_SIZE: "100" # The size of the boot disk in GB. +NUM_SOCKETS: "1" # The number of sockets for the VMs. +NUM_CORES: "4" # The number of cores for the VMs. +MEMORY_MIB: "8192" # The memory size for the VMs. + +EXP_CLUSTER_RESOURCE_SET: "true" # This enables the ClusterResourceSet feature that we are using to deploy CNI +CLUSTER_TOPOLOGY: "true" # This enables experimental ClusterClass templating \ No newline at end of file diff --git a/packages/apps/kubernetes-proxmox/templates/csi/deploy.yaml b/packages/apps/kubernetes-proxmox/templates/csi/deploy.yaml new file mode 100644 index 00000000..6d8783cf --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/csi/deploy.yaml @@ -0,0 +1,126 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-kcsi-controller + labels: + app: {{ .Release.Name }}-kcsi-driver +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-kcsi-driver + template: + metadata: + labels: + app: {{ .Release.Name }}-kcsi-driver + spec: + serviceAccountName: {{ .Release.Name }}-kcsi + priorityClassName: system-cluster-critical + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + effect: "NoSchedule" + containers: + - name: csi-driver + imagePullPolicy: Always + image: ghcr.io/kvaps/test:kubevirt-csi-driver + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--infra-cluster-namespace=$(INFRACLUSTER_NAMESPACE)" + - "--infra-cluster-labels=$(INFRACLUSTER_LABELS)" + - "--v=5" + ports: + - name: healthz + containerPort: 10301 + protocol: TCP + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: INFRACLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: INFRACLUSTER_LABELS + value: "csi-driver/cluster=test" + - name: INFRA_STORAGE_CLASS_ENFORCEMENT + valueFrom: + configMapKeyRef: + name: driver-config + key: infraStorageClassEnforcement + optional: true + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + resources: + requests: + memory: 50Mi + cpu: 10m + - name: csi-provisioner + image: quay.io/openshift/origin-csi-external-provisioner:latest + args: + - "--csi-address=$(ADDRESS)" + - "--default-fstype=ext4" + - "--kubeconfig=/etc/kubernetes/kubeconfig/super-admin.svc" + - "--v=5" + - "--timeout=3m" + - "--retry-interval-max=1m" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + - name: csi-attacher + image: quay.io/openshift/origin-csi-external-attacher:latest + args: + - "--csi-address=$(ADDRESS)" + - "--kubeconfig=/etc/kubernetes/kubeconfig/super-admin.svc" + - "--v=5" + - "--timeout=3m" + - "--retry-interval-max=1m" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + resources: + requests: + memory: 50Mi + cpu: 10m + - name: csi-liveness-probe + image: quay.io/openshift/origin-csi-livenessprobe:latest + args: + - "--csi-address=/csi/csi.sock" + - "--probe-timeout=3s" + - "--health-port=10301" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + requests: + memory: 50Mi + cpu: 10m + volumes: + - name: socket-dir + emptyDir: {} + - secret: + secretName: {{ .Release.Name }}-admin-kubeconfig + name: kubeconfig diff --git a/packages/apps/kubernetes-proxmox/templates/csi/infra-cluster-service-account.yaml b/packages/apps/kubernetes-proxmox/templates/csi/infra-cluster-service-account.yaml new file mode 100644 index 00000000..d70ea04a --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/csi/infra-cluster-service-account.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-kcsi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-kcsi +rules: +- apiGroups: ["cdi.kubevirt.io"] + resources: ["datavolumes"] + verbs: ["get", "create", "delete"] +- apiGroups: ["kubevirt.io"] + resources: ["virtualmachineinstances"] + verbs: ["list", "get"] +- apiGroups: ["subresources.kubevirt.io"] + resources: ["virtualmachineinstances/addvolume", "virtualmachineinstances/removevolume"] + verbs: ["update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-kcsi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-kcsi +subjects: +- kind: ServiceAccount + name: {{ .Release.Name }}-kcsi diff --git a/packages/apps/kubernetes-proxmox/templates/helmreleases/cilium.yaml b/packages/apps/kubernetes-proxmox/templates/helmreleases/cilium.yaml new file mode 100644 index 00000000..96c759fd --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/helmreleases/cilium.yaml @@ -0,0 +1,46 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: {{ .Release.Name }}-cilium + labels: + cozystack.io/repository: system + coztstack.io/target-cluster-name: {{ .Release.Name }} +spec: + interval: 1m + releaseName: cilium + chart: + spec: + chart: cozy-cilium + reconcileStrategy: Revision + sourceRef: + kind: HelmRepository + name: cozystack-system + namespace: cozy-system + kubeConfig: + secretRef: + name: {{ .Release.Name }}-kubeconfig + targetNamespace: cozy-cilium + storageNamespace: cozy-cilium + install: + createNamespace: true + values: + cilium: + tunnel: disabled + autoDirectNodeRoutes: true + cgroup: + autoMount: + enabled: true + hostRoot: /run/cilium/cgroupv2 + k8sServiceHost: {{ .Release.Name }}.{{ .Release.Namespace }}.svc + k8sServicePort: 6443 + + cni: + chainingMode: ~ + customConf: false + configMap: "" + routingMode: native + enableIPv4Masquerade: true + ipv4NativeRoutingCIDR: "10.244.0.0/16" + dependsOn: + - name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} diff --git a/packages/apps/kubernetes-proxmox/templates/helmreleases/csi.yaml b/packages/apps/kubernetes-proxmox/templates/helmreleases/csi.yaml new file mode 100644 index 00000000..55169278 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/helmreleases/csi.yaml @@ -0,0 +1,28 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: {{ .Release.Name }}-csi + labels: + cozystack.io/repository: system + coztstack.io/target-cluster-name: {{ .Release.Name }} +spec: + interval: 1m + releaseName: csi + chart: + spec: + chart: cozy-kubevirt-csi-node + reconcileStrategy: Revision + sourceRef: + kind: HelmRepository + name: cozystack-system + namespace: cozy-system + kubeConfig: + secretRef: + name: {{ .Release.Name }}-kubeconfig + targetNamespace: cozy-csi + storageNamespace: cozy-csi + install: + createNamespace: true + dependsOn: + - name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} diff --git a/packages/apps/kubernetes-proxmox/templates/helmreleases/delete.yaml b/packages/apps/kubernetes-proxmox/templates/helmreleases/delete.yaml new file mode 100644 index 00000000..e143653a --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/helmreleases/delete.yaml @@ -0,0 +1,73 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-weight": "10" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed + name: {{ .Release.Name }}-flux-teardown +spec: + template: + spec: + serviceAccountName: {{ .Release.Name }}-flux-teardown + restartPolicy: Never + containers: + - name: kubectl + image: docker.io/clastix/kubectl:v1.29.1 + command: + - kubectl + - --namespace={{ .Release.Namespace }} + - patch + - helmrelease + - {{ .Release.Name }}-cilium + - {{ .Release.Name }}-csi + - -p + - '{"spec": {"suspend": true}}' + - --type=merge +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-flux-teardown + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-delete-policy: before-hook-creation,hook-failed + helm.sh/hook-weight: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + "helm.sh/hook": pre-install,post-install,pre-delete + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed + "helm.sh/hook-weight": "5" + name: {{ .Release.Name }}-flux-teardown +rules: + - apiGroups: + - "helm.toolkit.fluxcd.io" + resources: + - helmreleases + verbs: + - get + - patch + resourceNames: + - {{ .Release.Name }}-cilium + - {{ .Release.Name }}-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation,hook-failed + helm.sh/hook-weight: "5" + name: {{ .Release.Name }}-flux-teardown +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-flux-teardown +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-flux-teardown + namespace: {{ .Release.Namespace }} diff --git a/packages/apps/kubernetes-proxmox/templates/pccm/manager.yaml b/packages/apps/kubernetes-proxmox/templates/pccm/manager.yaml new file mode 100644 index 00000000..4e0984b5 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/pccm/manager.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-kccm + labels: + helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 + app.kubernetes.io/name: {{ .Release.Name }}-kccm + app.kubernetes.io/instance: {{ .Release.Name }}-kccm + app.kubernetes.io/version: "v0.4.0" + app.kubernetes.io/managed-by: Helm + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: {{ .Release.Name }}-kccm + app.kubernetes.io/instance: {{ .Release.Name }}-kccm + spec: + enableServiceLinks: false + priorityClassName: system-cluster-critical + serviceAccountName: {{ .Release.Name }}-pccm + securityContext: + fsGroup: 10258 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 10258 + runAsNonRoot: true + runAsUser: 10258 + hostAliases: + [] + initContainers: + [] + containers: + - name: proxmox-cloud-controller-manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + image: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager + imagePullPolicy: IfNotPresent + args: + - --v=4 + - --cloud-provider=proxmox + - --cloud-config=/etc/cloud/cloud-config + - --controllers=cloud-node,cloud-node-lifecycle + - --leader-elect-resource-name=cloud-controller-manager-proxmox + - --use-service-account-credentials + - --secure-port=10258 + - --kubeconfig=/etc/kubernetes/kubeconfig/super-admin.svc + livenessProbe: + httpGet: + path: /healthz + port: 10258 + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 5 + resources: + requests: + cpu: 10m + memory: 32Mi + volumeMounts: + - mountPath: /etc/kubernetes/kubeconfig + name: kubeconfig + readOnly: true + - mountPath: /etc/proxmox + name: cloud-config + readOnly: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ .Release.Name }}-kccm + app.kubernetes.io/instance: {{ .Release.Name }}-kccm + volumes: + - name: cloud-config + secret: + secretName: {{ .Release.Name }}-cloud-config + defaultMode: 416 + - secret: + secretName: {{ .Release.Name }}-admin-kubeconfig + name: kubeconfig diff --git a/packages/apps/kubernetes-proxmox/templates/pccm/pccm_role.yaml b/packages/apps/kubernetes-proxmox/templates/pccm/pccm_role.yaml new file mode 100644 index 00000000..1124ae06 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/pccm/pccm_role.yaml @@ -0,0 +1,57 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Namespace }}-{{ .Release.Name }}-pccm + labels: + helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 + app.kubernetes.io/name: {{ .Release.Name }}-kccm + app.kubernetes.io/instance: {{ .Release.Name }}-kccm + app.kubernetes.io/version: "v0.4.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create diff --git a/packages/apps/kubernetes-proxmox/templates/pccm/pccm_role_binding.yaml b/packages/apps/kubernetes-proxmox/templates/pccm/pccm_role_binding.yaml new file mode 100644 index 00000000..a16e0129 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/pccm/pccm_role_binding.yaml @@ -0,0 +1,27 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Namespace }}-{{ .Release.Name }}-pccm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Namespace }}-{{ .Release.Name }}-pccm +subjects: +- kind: ServiceAccount + name: {{ .Release.Namespace }}-{{ .Release.Name }}-pccm + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Namespace }}-{{ .Release.Name }}-pccm:extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: {{ .Release.Namespace }}-{{ .Release.Name }}-pccm + namespace: {{ .Release.Namespace }} diff --git a/packages/apps/kubernetes-proxmox/templates/pccm/pccm_secrets.yaml b/packages/apps/kubernetes-proxmox/templates/pccm/pccm_secrets.yaml new file mode 100644 index 00000000..4014f9d1 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/pccm/pccm_secrets.yaml @@ -0,0 +1,11 @@ +{{- if ne (len .Values.config.clusters) 0 }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Namespace }}-{{ .Release.Name }}-pccm + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: {{ toYaml .Values.config | b64enc | quote }} +{{- end }} \ No newline at end of file diff --git a/packages/apps/kubernetes-proxmox/templates/pccm/service_account.yaml b/packages/apps/kubernetes-proxmox/templates/pccm/service_account.yaml new file mode 100644 index 00000000..fc25bc04 --- /dev/null +++ b/packages/apps/kubernetes-proxmox/templates/pccm/service_account.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-pccm + labels: + helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 + app.kubernetes.io/name: {{ .Release.Name }}-pccm + app.kubernetes.io/instance: {{ .Release.Name }}-pccm + app.kubernetes.io/version: "v0.4.0" + app.kubernetes.io/managed-by: Helm diff --git a/packages/apps/kubernetes-proxmox/values.yaml b/packages/apps/kubernetes-proxmox/values.yaml new file mode 100644 index 00000000..2a8fd46b --- /dev/null +++ b/packages/apps/kubernetes-proxmox/values.yaml @@ -0,0 +1,10 @@ +host: "" +controlPlane: + replicas: 2 +nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + resources: + cpu: 2 + memory: 1024Mi diff --git a/packages/core/installer/images/cozystack.tag b/packages/core/installer/images/cozystack.tag index 602d7b26..aff03e70 100644 --- a/packages/core/installer/images/cozystack.tag +++ b/packages/core/installer/images/cozystack.tag @@ -1 +1 @@ -ghcr.io/aenix-io/cozystack/cozystack:latest +ghcr.io/aenix-io/cozystack/cozystack:v0.9.0 diff --git a/packages/core/platform/bundles/paas-proxmox.yaml b/packages/core/platform/bundles/paas-proxmox.yaml new file mode 100644 index 00000000..dd82de12 --- /dev/null +++ b/packages/core/platform/bundles/paas-proxmox.yaml @@ -0,0 +1,138 @@ +{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }} + +releases: +- name: cilium + releaseName: cilium + chart: cozy-cilium + namespace: cozy-cilium + privileged: true + dependsOn: [] + +- name: kubeovn + releaseName: kubeovn + chart: cozy-kubeovn + namespace: cozy-kubeovn + privileged: true + dependsOn: [cilium] + values: + cozystack: + nodesHash: {{ include "cozystack.master-node-ips" . | sha256sum }} + kube-ovn: + ipv4: + POD_CIDR: "{{ index $cozyConfig.data "ipv4-pod-cidr" }}" + POD_GATEWAY: "{{ index $cozyConfig.data "ipv4-pod-gateway" }}" + SVC_CIDR: "{{ index $cozyConfig.data "ipv4-svc-cidr" }}" + JOIN_CIDR: "{{ index $cozyConfig.data "ipv4-join-cidr" }}" + +- name: cert-manager + releaseName: cert-manager + chart: cozy-cert-manager + namespace: cozy-cert-manager + dependsOn: [cilium,kubeovn] + +- name: cert-manager-issuers + releaseName: cert-manager-issuers + chart: cozy-cert-manager-issuers + namespace: cozy-cert-manager + dependsOn: [cilium,kubeovn,cert-manager] + +- name: victoria-metrics-operator + releaseName: victoria-metrics-operator + chart: cozy-victoria-metrics-operator + namespace: cozy-victoria-metrics-operator + dependsOn: [cilium,kubeovn,cert-manager] + +- name: monitoring + releaseName: monitoring + chart: cozy-monitoring + namespace: cozy-monitoring + privileged: true + dependsOn: [cilium,kubeovn,victoria-metrics-operator] + +- name: grafana-operator + releaseName: grafana-operator + chart: cozy-grafana-operator + namespace: cozy-grafana-operator + dependsOn: [cilium,kubeovn] + +- name: mariadb-operator + releaseName: mariadb-operator + chart: cozy-mariadb-operator + namespace: cozy-mariadb-operator + dependsOn: [cilium,kubeovn,cert-manager,victoria-metrics-operator] + +- name: postgres-operator + releaseName: postgres-operator + chart: cozy-postgres-operator + namespace: cozy-postgres-operator + dependsOn: [cilium,kubeovn,cert-manager] + +- name: rabbitmq-operator + releaseName: rabbitmq-operator + chart: cozy-rabbitmq-operator + namespace: cozy-rabbitmq-operator + dependsOn: [cilium,kubeovn] + +- name: redis-operator + releaseName: redis-operator + chart: cozy-redis-operator + namespace: cozy-redis-operator + dependsOn: [cilium,kubeovn] + +- name: proxmox-csi-operator + releaseName: proxmox-csi-operator + chart: cozy-proxmox-csi-operator + namespace: cozy-proxmox + dependsOn: [cilium,kubeovn,cert-manager] + +- name: proxmox-csi-operator + releaseName: proxmox-csi-operator + chart: cozy-proxmox-csi-operator + namespace: cozy-proxmox + dependsOn: [cilium,kubeovn,cert-manager] + +- name: telepresence + releaseName: traffic-manager + chart: cozy-telepresence + namespace: cozy-telepresence + dependsOn: [cilium,kubeovn] + +- name: dashboard + releaseName: dashboard + chart: cozy-dashboard + namespace: cozy-dashboard + dependsOn: [cilium,kubeovn] + {{- if .Capabilities.APIVersions.Has "source.toolkit.fluxcd.io/v1beta2" }} + {{- with (lookup "source.toolkit.fluxcd.io/v1beta2" "HelmRepository" "cozy-public" "").items }} + values: + kubeapps: + redis: + master: + podAnnotations: + {{- range $index, $repo := . }} + {{- with (($repo.status).artifact).revision }} + repository.cozystack.io/{{ $repo.metadata.name }}: {{ quote . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + +- name: kamaji + releaseName: kamaji + chart: cozy-kamaji + namespace: cozy-kamaji + dependsOn: [cilium,kubeovn,cert-manager] + +- name: capi-operator + releaseName: capi-operator + chart: cozy-capi-operator + namespace: cozy-cluster-api + privileged: true + dependsOn: [cilium,kubeovn,cert-manager] + +- name: capi-providers + releaseName: capi-providers + chart: cozy-capi-providers + namespace: cozy-cluster-api + privileged: true + dependsOn: [cilium,kubeovn,capi-operator] \ No newline at end of file diff --git a/packages/extra/etcd/templates/datastore.yaml b/packages/extra/etcd/templates/datastore.yaml index 591f6519..40c7bf6d 100644 --- a/packages/extra/etcd/templates/datastore.yaml +++ b/packages/extra/etcd/templates/datastore.yaml @@ -47,4 +47,4 @@ metadata: name: etcd-client-tls annotations: helm.sh/hook: pre-install - helm.sh/resource-policy: keep \ No newline at end of file + helm.sh/resource-policy: keep diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index 63e386c4..f2ce1076 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -18,6 +18,14 @@ cilium: k8sServiceHost: localhost k8sServicePort: 7445 +# cni: +# chainingMode: generic-veth +# customConf: true +# configMap: cni-configuration +# routingMode: native +# enableIPv4Masquerade: false +# enableIdentityMark: false + cni: chainingMode: generic-veth customConf: true diff --git a/packages/system/kubemox/.helmignore b/packages/system/kubemox/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/packages/system/kubemox/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packages/system/kubemox/Chart.yaml b/packages/system/kubemox/Chart.yaml new file mode 100644 index 00000000..bf91962e --- /dev/null +++ b/packages/system/kubemox/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: kubemox +description: A Helm chart for Kubernetes +maintainers: + - name: Alp Eren Celik + email: celial18@itu.edu.tr +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.2" diff --git a/packages/system/kubemox/Makefile b/packages/system/kubemox/Makefile new file mode 100644 index 00000000..ff8c09c0 --- /dev/null +++ b/packages/system/kubemox/Makefile @@ -0,0 +1,11 @@ +NAME=kubemox +NAMESPACE=cozy-$(NAME) + +include ../../../scripts/package-system.mk + +update: + rm -rf templates + mkdir templates + export RELEASE=v0.0.6-rc2 && \ + wget https://github.com/kubevirt/kubevirt/releases/download/$${RELEASE}/kubevirt-cr.yaml -O templates/kubevirt-cr.yaml + sed -i 's/namespace: kubevirt/namespace: cozy-kubevirt/g' templates/kubevirt-cr.yaml diff --git a/packages/system/kubemox/templates/NOTES.txt b/packages/system/kubemox/templates/NOTES.txt new file mode 100644 index 00000000..5da4c2ce --- /dev/null +++ b/packages/system/kubemox/templates/NOTES.txt @@ -0,0 +1,12 @@ +* Thanks for installing the kubemox! + +* Kubemox aims managing Proxmox resources through Kubernetes objects. + +- To create a VirtualMachine from a template on Proxmox use one of the examples below: + + 1. kubectl create -f kubemox/samples/virtualmachineclone.yaml + + 2. kubectl create -f kubemox/samples/virtualmachinscratch.yaml + + +DISCLAIMER: Kubemox is still under development so please use accordingly! diff --git a/packages/system/kubemox/templates/_helpers.tpl b/packages/system/kubemox/templates/_helpers.tpl new file mode 100644 index 00000000..a0e950e0 --- /dev/null +++ b/packages/system/kubemox/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "kubemox.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kubemox.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kubemox.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "kubemox.labels" -}} +helm.sh/chart: {{ include "kubemox.chart" . }} +{{ include "kubemox.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kubemox.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kubemox.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kubemox.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "kubemox.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/packages/system/kubemox/templates/crds/containers.yaml b/packages/system/kubemox/templates/crds/containers.yaml new file mode 100644 index 00000000..07347492 --- /dev/null +++ b/packages/system/kubemox/templates/crds/containers.yaml @@ -0,0 +1,107 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: containers.proxmox.alperen.cloud +spec: + conversion: + strategy: None + group: proxmox.alperen.cloud + names: + kind: Container + listKind: ContainerList + plural: containers + singular: container + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Container is the Schema for the containers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ContainerSpec defines the desired state of Container + properties: + name: + description: Name is the name of the Container + type: string + nodeName: + description: NodeName is the name of the target node of Proxmox + type: string + template: + description: TemplateSpec of the source Container + properties: + cores: + description: Cores is the number of CPU cores + type: integer + disk: + description: Disks is the list of disks + items: + properties: + size: + description: Size is the size of the disk + type: integer + storage: + description: Storage is the name of the storage + type: string + type: + description: Type is the type of the disk + type: string + type: object + type: array + memory: + description: Memory is the amount of memory in MB + type: integer + name: + description: Name of the template + type: string + network: + description: Networks is the list of networks + items: + properties: + bridge: + description: Bridge is the name of the bridge + type: string + model: + description: Name is the name of the network + type: string + type: object + type: array + type: object + required: + - name + - nodeName + type: object + status: + description: ContainerStatus defines the observed state of Container + properties: + id: + type: integer + name: + type: string + node: + type: string + state: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + type: string + uptime: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/packages/system/kubemox/templates/crds/customcertificate.yaml b/packages/system/kubemox/templates/crds/customcertificate.yaml new file mode 100644 index 00000000..be744409 --- /dev/null +++ b/packages/system/kubemox/templates/crds/customcertificate.yaml @@ -0,0 +1,100 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"apiextensions.k8s.io/v1","kind":"CustomResourceDefinition","metadata":{"annotations":{"controller-gen.kubebuilder.io/version":"v0.12.0"},"name":"customcertificates.proxmox.alperen.cloud"},"spec":{"group":"proxmox.alperen.cloud","names":{"kind":"CustomCertificate","listKind":"CustomCertificateList","plural":"customcertificates","singular":"customcertificate"},"scope":"Namespaced","versions":[{"name":"v1alpha1","schema":{"openAPIV3Schema":{"description":"CustomCertificate is the Schema for the customcertificates API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"CustomCertificateSpec defines the desired state of CustomCertificate","properties":{"certManagerSpec":{"properties":{"commonName":{"type":"string"},"dnsNames":{"items":{"type":"string"},"type":"array"},"issuerRef":{"properties":{"group":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"}},"type":"object"},"secretName":{"type":"string"},"usages":{"items":{"type":"string"},"type":"array"}},"type":"object"},"nodeName":{"description":"INSERT ADDITIONAL SPEC FIELDS - desired state of cluster Important: Run \"make\" to regenerate code after modifying this file","type":"string"},"proxmoxCertSpec":{"properties":{"certificate":{"type":"string"},"force":{"type":"boolean"},"nodeName":{"type":"string"},"privateKey":{"type":"string"},"restartProxy":{"type":"boolean"}},"type":"object"}},"type":"object"},"status":{"description":"CustomCertificateStatus defines the observed state of CustomCertificate","properties":{"status":{"description":"INSERT ADDITIONAL STATUS FIELD - define observed state of cluster Important: Run \"make\" to regenerate code after modifying this file","type":"string"}},"type":"object"}},"type":"object"}},"served":true,"storage":true,"subresources":{"status":{}}}]}} + creationTimestamp: "2024-02-17T20:53:20Z" + generation: 3 + name: customcertificates.proxmox.alperen.cloud + resourceVersion: "3299203" + uid: b6cf98fa-a272-4b2f-b150-f49ab3b5f368 +spec: + conversion: + strategy: None + group: proxmox.alperen.cloud + names: + kind: CustomCertificate + listKind: CustomCertificateList + plural: customcertificates + singular: customcertificate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: CustomCertificate is the Schema for the customcertificates API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CustomCertificateSpec defines the desired state of CustomCertificate + properties: + certManagerSpec: + properties: + commonName: + type: string + dnsNames: + items: + type: string + type: array + issuerRef: + properties: + group: + type: string + kind: + type: string + name: + type: string + type: object + secretName: + type: string + usages: + items: + type: string + type: array + type: object + nodeName: + description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file' + type: string + proxmoxCertSpec: + properties: + certificate: + type: string + force: + type: boolean + nodeName: + type: string + privateKey: + type: string + restartProxy: + type: boolean + type: object + type: object + status: + description: CustomCertificateStatus defines the observed state of CustomCertificate + properties: + status: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/packages/system/kubemox/templates/crds/managedvirtualmachines.yaml b/packages/system/kubemox/templates/crds/managedvirtualmachines.yaml new file mode 100644 index 00000000..e49a61cc --- /dev/null +++ b/packages/system/kubemox/templates/crds/managedvirtualmachines.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: managedvirtualmachines.proxmox.alperen.cloud +spec: + group: proxmox.alperen.cloud + names: + kind: ManagedVirtualMachine + listKind: ManagedVirtualMachineList + plural: managedvirtualmachines + singular: managedvirtualmachine + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ManagedVirtualMachine is the Schema for the managedvirtualmachines + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ManagedVirtualMachineSpec defines the desired state of ManagedVirtualMachine + properties: + cores: + type: integer + disk: + type: integer + memory: + type: integer + name: + description: Foo is an example field of ManagedVirtualMachine. Edit + managedvirtualmachine_types.go to remove/update + type: string + nodeName: + type: string + required: + - cores + - disk + - memory + - name + - nodeName + type: object + status: + description: ManagedVirtualMachineStatus defines the observed state of + ManagedVirtualMachine + properties: + IPAddress: + type: string + OSInfo: + type: string + id: + type: integer + name: + type: string + node: + type: string + state: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + type: string + uptime: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packages/system/kubemox/templates/crds/storagedownloadurls.yaml b/packages/system/kubemox/templates/crds/storagedownloadurls.yaml new file mode 100644 index 00000000..9e2cfe15 --- /dev/null +++ b/packages/system/kubemox/templates/crds/storagedownloadurls.yaml @@ -0,0 +1,70 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: storagedownloadurls.proxmox.alperen.cloud +spec: + conversion: + strategy: None + group: proxmox.alperen.cloud + names: + kind: StorageDownloadURL + listKind: StorageDownloadURLList + plural: storagedownloadurls + singular: storagedownloadurl + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: StorageDownloadURL is the Schema for the storagedownloadurls + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: StorageDownloadURLSpec defines the desired state of StorageDownloadURL + properties: + checksum: + description: Optional fields + type: string + checksumAlgorithm: + type: string + compression: + type: string + content: + type: string + filename: + type: string + node: + type: string + storage: + type: string + url: + type: string + verifyCertificate: + type: boolean + required: + - content + - filename + - node + - storage + - url + type: object + status: + description: StorageDownloadURLStatus defines the observed state of StorageDownloadURL + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/packages/system/kubemox/templates/crds/virtualmachines.yaml b/packages/system/kubemox/templates/crds/virtualmachines.yaml new file mode 100644 index 00000000..bc75029d --- /dev/null +++ b/packages/system/kubemox/templates/crds/virtualmachines.yaml @@ -0,0 +1,156 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: virtualmachines.proxmox.alperen.cloud +spec: + group: proxmox.alperen.cloud + names: + kind: VirtualMachine + listKind: VirtualMachineList + plural: virtualmachines + singular: virtualmachine + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: VirtualMachine is the Schema for the virtualmachines API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VirtualMachineSpec defines the desired state of VirtualMachine + properties: + name: + description: Name is the name of the VM + type: string + nodeName: + description: NodeName is the name of the target node of Proxmox + type: string + template: + description: TemplateSpec of the source VM + properties: + cores: + description: Cores is the number of CPU cores + type: integer + disk: + description: Disks is the list of disks + items: + properties: + size: + description: Size is the size of the disk in GB + type: integer + storage: + description: Storage is the name of the storage + type: string + type: + description: Type is the type of the disk + type: string + required: + - size + - storage + - type + type: object + type: array + memory: + description: Memory is the amount of memory in MB + type: integer + name: + description: Name of the template + type: string + network: + description: Networks is the list of networks + items: + properties: + bridge: + description: Bridge is the name of the bridge + type: string + model: + description: Model is the model of the network card + type: string + required: + - bridge + - model + type: object + type: array + socket: + description: Socket is the number of CPU sockets + type: integer + type: object + vmSpec: + description: This field should be modified further + properties: + cores: + description: CPUs + type: integer + disk: + description: Disks is the list of disks + properties: + name: + type: string + value: + type: string + type: object + memory: + description: Memory is the amount of memory in MB + type: integer + network: + description: Networks is the list of networks + properties: + name: + type: string + value: + type: string + type: object + osImage: + description: OS Image + properties: + name: + type: string + value: + type: string + type: object + type: object + required: + - name + - nodeName + type: object + status: + description: VirtualMachineStatus defines the observed state of VirtualMachine + properties: + IPAddress: + type: string + OSInfo: + type: string + id: + type: integer + name: + type: string + node: + type: string + state: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + type: string + uptime: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packages/system/kubemox/templates/crds/virtualmachinesets.yaml b/packages/system/kubemox/templates/crds/virtualmachinesets.yaml new file mode 100644 index 00000000..966a1980 --- /dev/null +++ b/packages/system/kubemox/templates/crds/virtualmachinesets.yaml @@ -0,0 +1,110 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: virtualmachinesets.proxmox.alperen.cloud +spec: + group: proxmox.alperen.cloud + names: + kind: VirtualMachineSet + listKind: VirtualMachineSetList + plural: virtualmachinesets + singular: virtualmachineset + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: VirtualMachineSet is the Schema for the virtualmachinesets API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VirtualMachineSetSpec defines the desired state of VirtualMachineSet + properties: + nodeName: + type: string + replicas: + description: Foo is an example field of VirtualMachineSet. Edit virtualmachineset_types.go + to remove/update + type: integer + template: + properties: + cores: + description: Cores is the number of CPU cores + type: integer + disk: + description: Disks is the list of disks + items: + properties: + size: + description: Size is the size of the disk in GB + type: integer + storage: + description: Storage is the name of the storage + type: string + type: + description: Type is the type of the disk + type: string + required: + - size + - storage + - type + type: object + type: array + memory: + description: Memory is the amount of memory in MB + type: integer + name: + description: Name of the template + type: string + network: + description: Networks is the list of networks + items: + properties: + bridge: + description: Bridge is the name of the bridge + type: string + model: + description: Model is the model of the network card + type: string + required: + - bridge + - model + type: object + type: array + socket: + description: Socket is the number of CPU sockets + type: integer + type: object + required: + - nodeName + - replicas + type: object + status: + description: VirtualMachineSetStatus defines the observed state of VirtualMachineSet + properties: + condition: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + type: string + required: + - condition + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/packages/system/kubemox/templates/crds/virtualmachinesnapshotpolicies.yaml b/packages/system/kubemox/templates/crds/virtualmachinesnapshotpolicies.yaml new file mode 100644 index 00000000..638a841b --- /dev/null +++ b/packages/system/kubemox/templates/crds/virtualmachinesnapshotpolicies.yaml @@ -0,0 +1,104 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: virtualmachinesnapshotpolicies.proxmox.alperen.cloud +spec: + group: proxmox.alperen.cloud + names: + kind: VirtualMachineSnapshotPolicy + listKind: VirtualMachineSnapshotPolicyList + plural: virtualmachinesnapshotpolicies + singular: virtualmachinesnapshotpolicy + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: VirtualMachineSnapshotPolicy is the Schema for the virtualmachinesnapshotpolicies + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VirtualMachineSnapshotPolicySpec defines the desired state + of VirtualMachineSnapshotPolicy + properties: + labelSelector: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + namespaces: + items: + type: string + type: array + type: object + snapshotSchedule: + type: string + type: object + status: + description: VirtualMachineSnapshotPolicyStatus defines the observed state + of VirtualMachineSnapshotPolicy + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/packages/system/kubemox/templates/crds/virtualmachinesnapshots.yaml b/packages/system/kubemox/templates/crds/virtualmachinesnapshots.yaml new file mode 100644 index 00000000..8542cb4c --- /dev/null +++ b/packages/system/kubemox/templates/crds/virtualmachinesnapshots.yaml @@ -0,0 +1,69 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: virtualmachinesnapshots.proxmox.alperen.cloud +spec: + group: proxmox.alperen.cloud + names: + kind: VirtualMachineSnapshot + listKind: VirtualMachineSnapshotList + plural: virtualmachinesnapshots + singular: virtualmachinesnapshot + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: VirtualMachineSnapshot is the Schema for the virtualmachinesnapshots + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VirtualMachineSnapshotSpec defines the desired state of VirtualMachineSnapshot + properties: + snapshotName: + description: Name of the snapshot + type: string + timestamp: + description: Description of the snapshot + format: date-time + type: string + virtualMachineName: + description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file + Name of the virtual machine to take snapshot of' + type: string + required: + - virtualMachineName + type: object + status: + description: VirtualMachineSnapshotStatus defines the observed state of + VirtualMachineSnapshot + properties: + errorMessage: + description: Error message if the snapshot creation process failed + type: string + status: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file Possible values: "created", "failed"' + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/packages/system/kubemox/templates/deployment.yaml b/packages/system/kubemox/templates/deployment.yaml new file mode 100644 index 00000000..b246d9d6 --- /dev/null +++ b/packages/system/kubemox/templates/deployment.yaml @@ -0,0 +1,98 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubemox.fullname" . }} + labels: + {{- include "kubemox.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "kubemox.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "kubemox.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "kubemox.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + # livenessProbe: + # httpGet: + # path: / + # port: http + # readinessProbe: + # httpGet: + # path: / + # port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PROXMOX_ENDPOINT + value: {{ .Values.proxmox.endpoint }} + - name: PROXMOX_INSECURE_SKIP_TLS_VERIFY + value: {{ .Values.proxmox.insecureSkipTLSVerify | quote }} + {{- if .Values.proxmox.tokenID }} + - name: PROXMOX_TOKEN_ID + valueFrom: + secretKeyRef: + name: proxmox-credentials + key: tokenID + {{- end }} + {{- if .Values.proxmox.secret }} + - name: PROXMOX_SECRET + valueFrom: + secretKeyRef: + name: proxmox-credentials + key: secret + {{- end }} + {{- if .Values.proxmox.username }} + - name: PROXMOX_USERNAME + valueFrom: + secretKeyRef: + name: proxmox-credentials + key: username + {{- end }} + {{- if .Values.proxmox.password }} + - name: PROXMOX_PASSWORD + valueFrom: + secretKeyRef: + name: proxmox-credentials + key: password + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/packages/system/kubemox/templates/hpa.yaml b/packages/system/kubemox/templates/hpa.yaml new file mode 100644 index 00000000..7d634219 --- /dev/null +++ b/packages/system/kubemox/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "kubemox.fullname" . }} + labels: + {{- include "kubemox.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "kubemox.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/packages/system/kubemox/templates/ingress.yaml b/packages/system/kubemox/templates/ingress.yaml new file mode 100644 index 00000000..69d4c94a --- /dev/null +++ b/packages/system/kubemox/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "kubemox.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "kubemox.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/packages/system/kubemox/templates/role.yaml b/packages/system/kubemox/templates/role.yaml new file mode 100644 index 00000000..4481a3e9 --- /dev/null +++ b/packages/system/kubemox/templates/role.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubemox +rules: +# Custom Resources +- apiGroups: ["proxmox.alperen.cloud"] + resources: + - "managedvirtualmachines" + - "managedvirtualmachines/finalizers" + - "managedvirtualmachines/status" + - "virtualmachines" + - "virtualmachines/finalizers" + - "virtualmachines/status" + - "virtualmachinesets" + - "virtualmachinesets/finalizers" + - "virtualmachinesets/status" + - "virtualmachinesnapshots" + - "virtualmachinesnapshots/finalizers" + - "virtualmachinesnapshots/status" + - "virtualmachinesnapshotpolicies" + - "virtualmachinesnapshotpolicies/finalizers" + - "virtualmachinesnapshotpolicies/status" + - "containers" + - "containers/finalizers" + - "containers/status" + - "customcertificates" + - "customcertificates/finalizers" + - "customcertificates/status" + - "storagedownloadurls" + - "storagedownloadurls/finalizers" + - "storagedownloadurls/status" + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +# Events +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +## Leader election +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +## List CRDs +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + - get +# Certificates +- apiGroups: ["cert-manager.io"] + resources: ["certificates"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] \ No newline at end of file diff --git a/packages/system/kubemox/templates/rolebinding.yaml b/packages/system/kubemox/templates/rolebinding.yaml new file mode 100644 index 00000000..dc5f27aa --- /dev/null +++ b/packages/system/kubemox/templates/rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubemox +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubemox +subjects: +- kind: ServiceAccount + name: kubemox + namespace: {{ .Release.Namespace }} diff --git a/packages/system/kubemox/templates/secret.yaml b/packages/system/kubemox/templates/secret.yaml new file mode 100644 index 00000000..adfb86ce --- /dev/null +++ b/packages/system/kubemox/templates/secret.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +data: + {{- if .Values.proxmox.secret }} + secret: {{ .Values.proxmox.secret | b64enc }} + {{- end }} + {{- if .Values.proxmox.tokenID }} + tokenID: {{ .Values.proxmox.tokenID | b64enc }} + {{- end }} + {{- if .Values.proxmox.username }} + username: {{ .Values.proxmox.username | b64enc }} + {{- end }} + {{- if .Values.proxmox.password }} + password: {{ .Values.proxmox.password | b64enc }} + {{- end }} +kind: Secret +metadata: + name: proxmox-credentials \ No newline at end of file diff --git a/packages/system/kubemox/templates/service.yaml b/packages/system/kubemox/templates/service.yaml new file mode 100644 index 00000000..98b4138d --- /dev/null +++ b/packages/system/kubemox/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubemox.fullname" . }} + labels: + {{- include "kubemox.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: {{ .Values.service.portName }} + selector: + {{- include "kubemox.selectorLabels" . | nindent 4 }} diff --git a/packages/system/kubemox/templates/serviceMonitor.yaml b/packages/system/kubemox/templates/serviceMonitor.yaml new file mode 100644 index 00000000..7ed3a59d --- /dev/null +++ b/packages/system/kubemox/templates/serviceMonitor.yaml @@ -0,0 +1,47 @@ +{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kubemox.fullname" . }} + namespace: {{ default .Release.Namespace .Values.metrics.serviceMonitor.namespace | quote }} + labels: + {{- include "kubemox.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.selector }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.metrics.service.portName }} + {{- with .Values.metrics.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + path: /metrics + {{- with .Values.metrics.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "kubemox.selectorLabels" . | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/packages/system/kubemox/templates/serviceaccount.yaml b/packages/system/kubemox/templates/serviceaccount.yaml new file mode 100644 index 00000000..60981531 --- /dev/null +++ b/packages/system/kubemox/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubemox.serviceAccountName" . }} + labels: + {{- include "kubemox.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/packages/system/kubemox/templates/tests/test-connection.yaml b/packages/system/kubemox/templates/tests/test-connection.yaml new file mode 100644 index 00000000..c8021781 --- /dev/null +++ b/packages/system/kubemox/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "kubemox.fullname" . }}-test-connection" + labels: + {{- include "kubemox.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "kubemox.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/packages/system/kubemox/values.yaml b/packages/system/kubemox/values.yaml new file mode 100644 index 00000000..3c653a04 --- /dev/null +++ b/packages/system/kubemox/values.yaml @@ -0,0 +1,126 @@ +# Default values for kubemox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +proxmox: + endpoint: "10.0.0.99" + # endpoint: 10.0.0.99 + # insecureSkipTLSVerify: true + insecureSkipTLSVerify: true + # Either tokenID/secret or username/password must be set + tokenID: "" + secret: "" + # Both username and password must be set if tokenID/secret is not set + username: "root@pam" + password: "PROXMOX_PASSWORD" + +image: + repository: alperencelik/kubemox + pullPolicy: Always + # Overrides the image tag whose default is the chart appVersion. + # See https://github.com/alperencelik/kubemox/releases for the tags + tag: "latest" + +metrics: + serviceMonitor: + enabled: false + interval: 30s + # -- Prometheus [MetricRelabelConfigs] to apply to samples before ingestion + relabelings: [] + # -- Prometheus [MetricRelabelConfigs] to apply to samples before ingestion + metricRelabelings: [] + # -- Prometheus ServiceMonitor selector + selector: {} + # prometheus: kube-prometheus + + # -- Prometheus ServiceMonitor scheme + scheme: "" + # -- Prometheus ServiceMonitor tlsConfig + tlsConfig: {} + # -- Prometheus ServiceMonitor namespace + namespace: "" # "monitoring" + # -- Prometheus ServiceMonitor labels + additionalLabels: {} + # -- Prometheus ServiceMonitor annotations + annotations: {} + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 8080 + targetPort: 8080 + portName: http + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/packages/system/proxmox-csi-node/Chart.yaml b/packages/system/proxmox-csi-node/Chart.yaml new file mode 100644 index 00000000..2a63eb08 --- /dev/null +++ b/packages/system/proxmox-csi-node/Chart.yaml @@ -0,0 +1,2 @@ +name: cozy-proxmox-csi-node +version: 0.1.0 diff --git a/packages/system/proxmox-csi-node/templates/deploy.yaml b/packages/system/proxmox-csi-node/templates/deploy.yaml new file mode 100644 index 00000000..52b8e130 --- /dev/null +++ b/packages/system/proxmox-csi-node/templates/deploy.yaml @@ -0,0 +1,278 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.kubevirt.io +spec: + attachRequired: true + podInfoOnMount: true + fsGroupPolicy: ReadWriteOnceWithFSType +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-csi-controller-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxmox-csi-controller-cr +rules: + - apiGroups: [''] + resources: ['persistentvolumes'] + verbs: ['create', 'delete', 'get', 'list', 'watch', 'update', 'patch'] + - apiGroups: [''] + resources: ['secrets'] + verbs: ['get', 'list'] + - apiGroups: [''] + resources: ['persistentvolumeclaims'] + verbs: ['get', 'list', 'watch', 'update'] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [''] + resources: ['nodes'] + verbs: ['get', 'list', 'watch'] + - apiGroups: ['storage.k8s.io'] + resources: ['volumeattachments'] + verbs: ['get', 'list', 'watch', 'update', 'patch'] + - apiGroups: ['storage.k8s.io'] + resources: ['storageclasses'] + verbs: ['get', 'list', 'watch'] + - apiGroups: ['csi.storage.k8s.io'] + resources: ['csidrivers'] + verbs: ['get', 'list', 'watch', 'update', 'create'] + - apiGroups: [''] + resources: ['events'] + verbs: ['list', 'watch', 'create', 'update', 'patch'] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments/status" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + verbs: ["use"] + resourceNames: ["privileged"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: proxmox-csi-controller-binding +subjects: + - kind: ServiceAccount + name: proxmox-csi-controller-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: proxmox-csi-controller-cr + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-csi-node-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxmox-csi-node-cr +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + verbs: ["use"] + resourceNames: ["privileged"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: proxmox-csi-node-binding +subjects: + - kind: ServiceAccount + name: proxmox-csi-node-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: proxmox-csi-node-cr + apiGroup: rbac.authorization.k8s.io +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: proxmox-csi-node + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + app: {{ .Release.Namespace }} + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: {{ .Release.Namespace }} + spec: + serviceAccount: proxmox-csi-node-sa + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: csi-driver + securityContext: + privileged: true + allowPrivilegeEscalation: true + imagePullPolicy: Always + image: ghcr.io/kvaps/test:proxmox-csi-driver + args: + - "--endpoint=unix:/csi/csi.sock" + - "--node-name=$(KUBE_NODE_NAME)" + - "--v=5" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + - name: udev + mountPath: /run/udev + ports: + - name: healthz + containerPort: 10300 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + resources: + requests: + memory: 50Mi + cpu: 10m + - name: csi-node-driver-registrar + image: quay.io/openshift/origin-csi-node-driver-registrar:latest + args: + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + - "--v=5" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi.kubevirt.io-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.proxmox.io/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + requests: + memory: 20Mi + cpu: 5m + - name: csi-liveness-probe + image: quay.io/openshift/origin-csi-livenessprobe:latest + args: + - "--csi-address=/csi/csi.sock" + - "--probe-timeout=3s" + - "--health-port=10300" + volumeMounts: + - name: plugin-dir + mountPath: /csi + resources: + requests: + memory: 20Mi + cpu: 5m + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.proxmox.io/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: udev + hostPath: + path: /run/udev +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: driver-config + namespace: {{ .Release.Namespace }} +data: + infraClusterNamespace: + infraClusterLabels: csi-driver/cluster=tenant +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: proxmox + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: csi.proxmox.io +parameters: + infraStorageClassName: replicated + bus: scsi diff --git a/packages/system/proxmox-csi/Chart.yaml b/packages/system/proxmox-csi/Chart.yaml new file mode 100644 index 00000000..0aa33800 --- /dev/null +++ b/packages/system/proxmox-csi/Chart.yaml @@ -0,0 +1,2 @@ +name: app +version: 0.0.0 diff --git a/packages/system/proxmox-csi/Makefile b/packages/system/proxmox-csi/Makefile new file mode 100644 index 00000000..357a739e --- /dev/null +++ b/packages/system/proxmox-csi/Makefile @@ -0,0 +1,13 @@ +include ../../hack/app-helm.mk + +update: + rm -rf charts + tag=$$(git ls-remote --tags --sort="v:refname" https://github.com/sergelogvinov/proxmox-cloud-controller-manager | awk -F'[/^]' 'END{print $$3}') && \ + curl -sSL https://github.com/sergelogvinov/proxmox-cloud-controller-manager/archive/refs/tags/$${tag}.tar.gz | \ + tar xzvf - --strip 1 proxmox-cloud-controller-manager-$${tag#*v}/charts + sed -i 's/^ namespace: .*/ namespace: kube-system/' charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml + tag=$$(git ls-remote --tags --sort="v:refname" https://github.com/sergelogvinov/proxmox-csi-plugin | awk -F'[/^]' 'END{print $$3}') && \ + curl -sSL https://github.com/sergelogvinov/proxmox-csi-plugin/archive/refs/tags/$${tag}.tar.gz | \ + tar xzvf - --strip 1 proxmox-csi-plugin-$${tag#*v}/charts + rm -f charts/proxmox-csi-plugin/templates/namespace.yaml + patch -p 3 < patches/namespace.patch diff --git a/packages/system/proxmox-csi/README.md b/packages/system/proxmox-csi/README.md new file mode 100644 index 00000000..d95c5324 --- /dev/null +++ b/packages/system/proxmox-csi/README.md @@ -0,0 +1,6 @@ +# Proxmox CSI Plugin + +Plugin that provides CSI interface for Proxmox + +- GitHub: https://github.com/sergelogvinov/proxmox-csi-plugin +- Telegram: https://t.me/ru_talos diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/.helmignore b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/Chart.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/Chart.yaml new file mode 100644 index 00000000..cd2b2ebd --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: proxmox-cloud-controller-manager +description: A Helm chart for Kubernetes +type: application +home: https://github.com/sergelogvinov/proxmox-cloud-controller-manager +icon: https://proxmox.com/templates/yoo_nano2/favicon.ico +sources: +- https://github.com/sergelogvinov/proxmox-cloud-controller-manager +keywords: +- ccm +maintainers: +- name: sergelogvinov + url: https://github.com/sergelogvinov + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.6 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: v0.2.0 diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md new file mode 100644 index 00000000..94a97015 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md @@ -0,0 +1,81 @@ +# proxmox-cloud-controller-manager + +![Version: 0.1.6](https://img.shields.io/badge/Version-0.1.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.2.0](https://img.shields.io/badge/AppVersion-v0.2.0-informational?style=flat-square) + +A Helm chart for Kubernetes + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| sergelogvinov | | | + +## Source Code + +* + +Example: + +```yaml +# proxmox-ccm.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes@pve!csi" + token_secret: "key" + region: cluster-1 + +enabledControllers: + # Remove `cloud-node` if you use it with Talos CCM + - cloud-node + - cloud-node-lifecycle + +# Deploy CCM only on control-plane nodes +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +``` + +Deploy chart: + +```shell +helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ + proxmox-cloud-controller-manager charts/proxmox-cloud-controller-manager +``` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| replicaCount | int | `1` | | +| image.repository | string | `"ghcr.io/sergelogvinov/proxmox-cloud-controller-manager"` | Proxmox CCM image. | +| image.pullPolicy | string | `"IfNotPresent"` | Always or IfNotPresent | +| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | +| imagePullSecrets | list | `[]` | | +| nameOverride | string | `""` | | +| fullnameOverride | string | `""` | | +| extraArgs | list | `[]` | Any extra arguments for talos-cloud-controller-manager | +| enabledControllers | list | `["cloud-node","cloud-node-lifecycle"]` | List of controllers should be enabled. Use '*' to enable all controllers. Support only `cloud-node,cloud-node-lifecycle` controllers. | +| logVerbosityLevel | int | `2` | Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md for description of individual verbosity levels. | +| existingConfigSecret | string | `nil` | Proxmox cluster config stored in secrets. | +| existingConfigSecretKey | string | `"config.yaml"` | Proxmox cluster config stored in secrets key. | +| config | object | `{"clusters":[]}` | Proxmox cluster config. | +| serviceAccount | object | `{"annotations":{},"create":true,"name":""}` | Pods Service Account. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ | +| priorityClassName | string | `"system-cluster-critical"` | CCM pods' priorityClassName. | +| podAnnotations | object | `{}` | Annotations for data pods. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | +| podSecurityContext | object | `{"fsGroup":10258,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":10258,"runAsNonRoot":true,"runAsUser":10258}` | Pods Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"seccompProfile":{"type":"RuntimeDefault"}}` | Container Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| resources | object | `{"requests":{"cpu":"10m","memory":"32Mi"}}` | Resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | Deployment update stategy type. ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment | +| nodeSelector | object | `{}` | Node labels for data pods assignment. ref: https://kubernetes.io/docs/user-guide/node-selection/ | +| tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Exists"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","operator":"Exists"}]` | Tolerations for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | +| affinity | object | `{}` | Affinity for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.11.2](https://github.com/norwoodj/helm-docs/releases/v1.11.2) diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md.gotmpl b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md.gotmpl new file mode 100644 index 00000000..b07e90dc --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md.gotmpl @@ -0,0 +1,52 @@ +{{ template "chart.header" . }} + +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +Example: + +```yaml +# proxmox-ccm.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes@pve!csi" + token_secret: "key" + region: cluster-1 + +enabledControllers: + # Remove `cloud-node` if you use it with Talos CCM + - cloud-node + - cloud-node-lifecycle + +# Deploy CCM only on control-plane nodes +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +``` + +Deploy chart: + +```shell +helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ + proxmox-cloud-controller-manager charts/proxmox-cloud-controller-manager +``` + +{{ template "chart.valuesSection" . }} + +{{ template "helm-docs.versionFooter" . }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/ci/values.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/ci/values.yaml new file mode 100644 index 00000000..8e61cb01 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/ci/values.yaml @@ -0,0 +1,27 @@ + +image: + repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager + pullPolicy: Always + tag: edge + +nodeSelector: + node-role.kubernetes.io/control-plane: "" + +logVerbosityLevel: 4 + +enabledControllers: + - cloud-node + - cloud-node-lifecycle + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-1 + - url: https://cluster-api-2.exmple.com:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-2 diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/NOTES.txt b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/NOTES.txt new file mode 100644 index 00000000..e69de29b diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl new file mode 100644 index 00000000..ee962923 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl @@ -0,0 +1,69 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "proxmox-cloud-controller-manager.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "proxmox-cloud-controller-manager.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "proxmox-cloud-controller-manager.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "proxmox-cloud-controller-manager.labels" -}} +helm.sh/chart: {{ include "proxmox-cloud-controller-manager.chart" . }} +{{ include "proxmox-cloud-controller-manager.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "proxmox-cloud-controller-manager.selectorLabels" -}} +app.kubernetes.io/name: {{ include "proxmox-cloud-controller-manager.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "proxmox-cloud-controller-manager.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "proxmox-cloud-controller-manager.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Generate string of enabled controllers. Might have a trailing comma (,) which needs to be trimmed. +*/}} +{{- define "proxmox-cloud-controller-manager.enabledControllers" }} +{{- range .Values.enabledControllers -}}{{ . }},{{- end -}} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/deployment.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/deployment.yaml new file mode 100644 index 00000000..10eea712 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/deployment.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.replicaCount }} + strategy: + type: {{ .Values.updateStrategy.type }} + selector: + matchLabels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- if .Values.config }} + checksum/config: {{ toJson .Values.config | sha256sum }} + {{- end }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 8 }} + spec: + enableServiceLinks: false + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --v={{ .Values.logVerbosityLevel }} + - --cloud-provider=proxmox + - --cloud-config=/etc/proxmox/config.yaml + - --controllers={{- trimAll "," (include "proxmox-cloud-controller-manager.enabledControllers" . ) }} + - --leader-elect-resource-name=cloud-controller-manager-proxmox + - --use-service-account-credentials + - --secure-port=10258 + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 10258 + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 5 + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: cloud-config + mountPath: /etc/proxmox + readOnly: true + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 14 }} + volumes: + {{- if .Values.existingConfigSecret }} + - name: cloud-config + secret: + secretName: {{ .Values.existingConfigSecret }} + items: + - key: {{ .Values.existingConfigSecretKey }} + path: config.yaml + defaultMode: 416 + {{- else }} + - name: cloud-config + secret: + secretName: {{ include "proxmox-cloud-controller-manager.fullname" . }} + defaultMode: 416 + {{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/role.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/role.yaml new file mode 100644 index 00000000..b35bf2b7 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/role.yaml @@ -0,0 +1,53 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml new file mode 100644 index 00000000..32b065ec --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml @@ -0,0 +1,26 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }}:extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/secrets.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/secrets.yaml new file mode 100644 index 00000000..ce60f622 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/secrets.yaml @@ -0,0 +1,11 @@ +{{- if ne (len .Values.config.clusters) 0 }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: {{ toYaml .Values.config | b64enc | quote }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml new file mode 100644 index 00000000..b9ecfcc2 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.edge.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.edge.yaml new file mode 100644 index 00000000..b90cc003 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.edge.yaml @@ -0,0 +1,13 @@ + +image: + pullPolicy: Always + tag: edge + +nodeSelector: + node-role.kubernetes.io/control-plane: "" + +logVerbosityLevel: 4 + +enabledControllers: + - cloud-node + - cloud-node-lifecycle diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.talos.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.talos.yaml new file mode 100644 index 00000000..2730ddf5 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.talos.yaml @@ -0,0 +1,8 @@ + +nodeSelector: + node-role.kubernetes.io/control-plane: "" + +logVerbosityLevel: 4 + +enabledControllers: + - cloud-node-lifecycle diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.yaml new file mode 100644 index 00000000..cf98befa --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.yaml @@ -0,0 +1,125 @@ +# Default values for proxmox-cloud-controller-manager. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + # -- Proxmox CCM image. + repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager + # -- Always or IfNotPresent + pullPolicy: IfNotPresent + # -- Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +# -- Any extra arguments for talos-cloud-controller-manager +extraArgs: [] + # - --cluster-name=kubernetes + +# -- List of controllers should be enabled. +# Use '*' to enable all controllers. +# Support only `cloud-node,cloud-node-lifecycle` controllers. +enabledControllers: + - cloud-node + - cloud-node-lifecycle + # - route + # - service + +# -- Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md +# for description of individual verbosity levels. +logVerbosityLevel: 2 + +# -- Proxmox cluster config stored in secrets. +existingConfigSecret: ~ +# -- Proxmox cluster config stored in secrets key. +existingConfigSecretKey: config.yaml + +# -- Proxmox cluster config. +config: + clusters: [] + # - url: https://cluster-api-1.exmple.com:8006/api2/json + # insecure: false + # token_id: "login!name" + # token_secret: "secret" + # region: cluster-1 + +# -- Pods Service Account. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# -- CCM pods' priorityClassName. +priorityClassName: system-cluster-critical + +# -- Annotations for data pods. +# ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + +# -- Pods Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +podSecurityContext: + runAsNonRoot: true + runAsUser: 10258 + runAsGroup: 10258 + fsGroup: 10258 + fsGroupChangePolicy: "OnRootMismatch" + +# -- Container Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# -- Resource requests and limits. +# ref: https://kubernetes.io/docs/user-guide/compute-resources/ +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + requests: + cpu: 10m + memory: 32Mi + +# -- Deployment update stategy type. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +# -- Node labels for data pods assignment. +# ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + # node-role.kubernetes.io/control-plane: "" + +# -- Tolerations for data pods assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + +# -- Affinity for data pods assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: {} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/.helmignore b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/Chart.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/Chart.yaml new file mode 100644 index 00000000..fd34e387 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: proxmox-csi-plugin +description: A CSI plugin for Proxmox +type: application +home: https://github.com/sergelogvinov/proxmox-csi-plugin +icon: https://proxmox.com/templates/yoo_nano2/favicon.ico +sources: +- https://github.com/sergelogvinov/proxmox-csi-plugin +keywords: +- storage +- block-storage +- volume +maintainers: +- name: sergelogvinov + url: https://github.com/sergelogvinov + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.6 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: v0.3.0 diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md new file mode 100644 index 00000000..c951a316 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md @@ -0,0 +1,116 @@ +# proxmox-csi-plugin + +![Version: 0.1.6](https://img.shields.io/badge/Version-0.1.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.3.0](https://img.shields.io/badge/AppVersion-v0.3.0-informational?style=flat-square) + +A CSI plugin for Proxmox + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| sergelogvinov | | | + +## Source Code + +* + +Example: + +```yaml +# proxmox-csi.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes-csi@pve!csi" + token_secret: "key" + region: cluster-1 + +# Deploy Node CSI driver only on proxmox nodes +node: + nodeSelector: + # It will work only with Talos CCM, remove it overwise + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +# Deploy CSI controller only on control-plane nodes +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +# Define storage classes +# See https://pve.proxmox.com/wiki/Storage +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete + fstype: ext4 + cache: writethrough +``` + +Deploy chart: + +```shell +helm upgrade -i --namespace=csi-proxmox -f proxmox-csi.yaml \ + proxmox-csi-plugin charts/proxmox-csi-plugin/ +``` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| replicaCount | int | `1` | | +| imagePullSecrets | list | `[]` | | +| nameOverride | string | `""` | | +| fullnameOverride | string | `""` | | +| priorityClassName | string | `"system-cluster-critical"` | Controller pods priorityClassName. | +| serviceAccount | object | `{"annotations":{},"create":true,"name":""}` | Pods Service Account. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ | +| provisionerName | string | `"csi.proxmox.sinextra.dev"` | CSI Driver provisioner name. Currently, cannot be customized. | +| clusterID | string | `"kubernetes"` | Cluster name. Currently, cannot be customized. | +| logVerbosityLevel | int | `5` | Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md for description of individual verbosity levels. | +| timeout | string | `"3m"` | Connection timeout between sidecars. | +| existingConfigSecret | string | `nil` | Proxmox cluster config stored in secrets. | +| existingConfigSecretKey | string | `"config.yaml"` | Proxmox cluster config stored in secrets key. | +| configFile | string | `"/etc/proxmox/config.yaml"` | Proxmox cluster config path. | +| config | object | `{"clusters":[]}` | Proxmox cluster config. | +| storageClass | list | `[]` | Storage class defenition. | +| controller.plugin.image | object | `{"pullPolicy":"IfNotPresent","repository":"ghcr.io/sergelogvinov/proxmox-csi-controller","tag":""}` | Controller CSI Driver. | +| controller.plugin.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Controller resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| controller.attacher.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-attacher","tag":"v4.3.0"}` | CSI Attacher. | +| controller.attacher.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Attacher resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| controller.provisioner.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-provisioner","tag":"v3.5.0"}` | CSI Provisioner. | +| controller.provisioner.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Provisioner resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| controller.resizer.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-resizer","tag":"v1.8.0"}` | CSI Resizer. | +| controller.resizer.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Resizer resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| node.plugin.image | object | `{"pullPolicy":"IfNotPresent","repository":"ghcr.io/sergelogvinov/proxmox-csi-node","tag":""}` | Node CSI Driver. | +| node.plugin.resources | object | `{}` | Node CSI Driver resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| node.driverRegistrar.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-node-driver-registrar","tag":"v2.8.0"}` | Node CSI driver registrar. | +| node.driverRegistrar.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Node registrar resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| node.nodeSelector | object | `{}` | Node labels for node-plugin assignment. ref: https://kubernetes.io/docs/user-guide/node-selection/ | +| node.tolerations | list | `[{"effect":"NoSchedule","key":"node.kubernetes.io/unschedulable","operator":"Exists"},{"effect":"NoSchedule","key":"node.kubernetes.io/disk-pressure","operator":"Exists"}]` | Tolerations for node-plugin assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | +| livenessprobe.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/livenessprobe","tag":"v2.10.0"}` | Common livenessprobe sidecar. | +| livenessprobe.failureThreshold | int | `5` | Failure threshold for livenessProbe | +| livenessprobe.initialDelaySeconds | int | `10` | Initial delay seconds for livenessProbe | +| livenessprobe.timeoutSeconds | int | `10` | Timeout seconds for livenessProbe | +| livenessprobe.periodSeconds | int | `60` | Period seconds for livenessProbe | +| livenessprobe.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Liveness probe resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| podAnnotations | object | `{}` | Annotations for controller pod. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | +| podSecurityContext | object | `{"fsGroup":65532,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | Controller Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Controller Container Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | Controller deployment update stategy type. ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment | +| nodeSelector | object | `{}` | Node labels for controller assignment. ref: https://kubernetes.io/docs/user-guide/node-selection/ | +| tolerations | list | `[]` | Tolerations for controller assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | +| affinity | object | `{}` | Affinity for controller assignment. ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md.gotmpl b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md.gotmpl new file mode 100644 index 00000000..da7ee0ce --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md.gotmpl @@ -0,0 +1,68 @@ +{{ template "chart.header" . }} + +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +Example: + +```yaml +# proxmox-csi.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes-csi@pve!csi" + token_secret: "key" + region: cluster-1 + +# Deploy Node CSI driver only on proxmox nodes +node: + nodeSelector: + # It will work only with Talos CCM, remove it overwise + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +# Deploy CSI controller only on control-plane nodes +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +# Define storage classes +# See https://pve.proxmox.com/wiki/Storage +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete + fstype: ext4 + cache: writethrough +``` + +Deploy chart: + +```shell +helm upgrade -i --namespace=csi-proxmox -f proxmox-csi.yaml \ + proxmox-csi-plugin charts/proxmox-csi-plugin/ +``` + +{{ template "chart.valuesSection" . }} + +{{ template "helm-docs.versionFooter" . }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/ci/values.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/ci/values.yaml new file mode 100644 index 00000000..7c834140 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/ci/values.yaml @@ -0,0 +1,22 @@ + +node: + nodeSelector: + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete + ssd: true diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/NOTES.txt b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/NOTES.txt new file mode 100644 index 00000000..e69de29b diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_helpers.tpl b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_helpers.tpl new file mode 100644 index 00000000..2a7f8027 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_helpers.tpl @@ -0,0 +1,71 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "proxmox-csi-plugin.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "proxmox-csi-plugin.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "proxmox-csi-plugin.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "proxmox-csi-plugin.labels" -}} +helm.sh/chart: {{ include "proxmox-csi-plugin.chart" . }} +app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "proxmox-csi-plugin.selectorLabels" -}} +app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: controller +{{- end }} + +{{- define "proxmox-csi-plugin-node.selectorLabels" -}} +app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: node +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "proxmox-csi-plugin.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "proxmox-csi-plugin.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-clusterrole.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-clusterrole.yaml new file mode 100644 index 00000000..6737e23c --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-clusterrole.yaml @@ -0,0 +1,37 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get","list", "watch", "create", "update", "patch"] + + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-deployment.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-deployment.yaml new file mode 100644 index 00000000..605a1f08 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-deployment.yaml @@ -0,0 +1,157 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + strategy: + type: {{ .Values.updateStrategy.type }} + rollingUpdate: + {{- toYaml .Values.updateStrategy.rollingUpdate | nindent 6 }} + selector: + matchLabels: + {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ toJson .Values.config | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 8 }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + enableServiceLinks: false + serviceAccountName: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.plugin.image.repository }}:{{ .Values.controller.plugin.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.controller.plugin.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--cloud-config={{ .Values.configFile }}" + resources: + {{- toYaml .Values.controller.plugin.resources | nindent 12 }} + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: cloud-config + mountPath: /etc/proxmox/ + - name: csi-attacher + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.attacher.image.repository }}:{{ .Values.controller.attacher.image.tag }}" + imagePullPolicy: {{ .Values.controller.attacher.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout={{ .Values.timeout }}" + - "--leader-election" + - "--default-fstype=ext4" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.controller.attacher.resources | nindent 12 }} + - name: csi-provisioner + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.provisioner.image.repository }}:{{ .Values.controller.provisioner.image.tag }}" + imagePullPolicy: {{ .Values.controller.provisioner.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout={{ .Values.timeout }}" + - "--leader-election" + - "--default-fstype=ext4" + - "--feature-gates=Topology=True" + - "--enable-capacity" + - "--capacity-ownerref-level=2" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.controller.provisioner.resources | nindent 12 }} + - name: csi-resizer + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.resizer.image.repository }}:{{ .Values.controller.resizer.image.tag }}" + imagePullPolicy: {{ .Values.controller.resizer.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout={{ .Values.timeout }}" + - "--handle-volume-inuse-error=false" + - "--leader-election" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.controller.resizer.resources | nindent 12 }} + - name: liveness-probe + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.livenessprobe.image.repository }}:{{ .Values.livenessprobe.image.tag }}" + imagePullPolicy: {{ .Values.livenessprobe.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.livenessprobe.resources | nindent 12 }} + volumes: + - name: socket-dir + emptyDir: {} + {{- if .Values.existingConfigSecret }} + - name: cloud-config + secret: + secretName: {{ .Values.existingConfigSecret }} + items: + - key: {{ .Values.existingConfigSecretKey }} + path: config.yaml + {{- else }} + - name: cloud-config + secret: + secretName: {{ include "proxmox-csi-plugin.fullname" . }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 14 }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-role.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-role.yaml new file mode 100644 index 00000000..b69d223f --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-role.yaml @@ -0,0 +1,21 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + + - apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-rolebinding.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-rolebinding.yaml new file mode 100644 index 00000000..478e0f6f --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-rolebinding.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + namespace: {{ .Release.Namespace }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/csidriver.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/csidriver.yaml new file mode 100644 index 00000000..5d0652e8 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/csidriver.yaml @@ -0,0 +1,10 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: {{ .Values.provisionerName }} +spec: + attachRequired: true + podInfoOnMount: true + storageCapacity: true + volumeLifecycleModes: + - Persistent diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-clusterrole.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-clusterrole.yaml new file mode 100644 index 00000000..02af491f --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-clusterrole.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-node + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-deployment.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-deployment.yaml new file mode 100644 index 00000000..deba6833 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-deployment.yaml @@ -0,0 +1,135 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-node + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +spec: + updateStrategy: + type: {{ .Values.updateStrategy.type }} + selector: + matchLabels: + {{- include "proxmox-csi-plugin-node.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "proxmox-csi-plugin-node.selectorLabels" . | nindent 8 }} + spec: + priorityClassName: system-node-critical + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + enableServiceLinks: false + serviceAccountName: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: {{ include "proxmox-csi-plugin.fullname" . }}-node + securityContext: + privileged: true + capabilities: + drop: + - ALL + add: + - SYS_ADMIN + - CHOWN + - DAC_OVERRIDE + seccompProfile: + type: RuntimeDefault + image: "{{ .Values.node.plugin.image.repository }}:{{ .Values.node.plugin.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.node.plugin.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--node-id=$(NODE_NAME)" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: {{- toYaml .Values.node.plugin.resources | nindent 12 }} + volumeMounts: + - name: socket + mountPath: /csi + - name: kubelet + mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + - name: dev + mountPath: /dev + - name: sys + mountPath: /sys + - name: csi-node-driver-registrar + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + # readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "{{ .Values.node.driverRegistrar.image.repository }}:{{ .Values.node.driverRegistrar.image.tag }}" + imagePullPolicy: {{ .Values.node.driverRegistrar.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/{{ .Values.provisionerName }}/csi.sock" + volumeMounts: + - name: socket + mountPath: /csi + - name: registration + mountPath: /registration + resources: {{- toYaml .Values.node.driverRegistrar.resources | nindent 12 }} + - name: liveness-probe + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "{{ .Values.livenessprobe.image.repository }}:{{ .Values.livenessprobe.image.tag }}" + imagePullPolicy: {{ .Values.livenessprobe.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + volumeMounts: + - name: socket + mountPath: /csi + resources: {{- toYaml .Values.livenessprobe.resources | nindent 12 }} + volumes: + - name: socket + hostPath: + path: /var/lib/kubelet/plugins/{{ .Values.provisionerName }}/ + type: DirectoryOrCreate + - name: registration + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: kubelet + hostPath: + path: /var/lib/kubelet + type: Directory + - name: dev + hostPath: + path: /dev + type: Directory + - name: sys + hostPath: + path: /sys + type: Directory + {{- with .Values.node.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.node.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-rolebinding.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-rolebinding.yaml new file mode 100644 index 00000000..4585dd66 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "proxmox-csi-plugin.fullname" . }}-node +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node + namespace: {{ .Release.Namespace }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/secrets.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/secrets.yaml new file mode 100644 index 00000000..f96f5d57 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/secrets.yaml @@ -0,0 +1,12 @@ +{{- if ne (len .Values.config.clusters) 0 }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +type: Opaque +data: + config.yaml: {{ toYaml .Values.config | b64enc | quote }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/serviceaccount.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/serviceaccount.yaml new file mode 100644 index 00000000..d6a45548 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/serviceaccount.yaml @@ -0,0 +1,25 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/storageclass.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/storageclass.yaml new file mode 100644 index 00000000..2049f620 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/storageclass.yaml @@ -0,0 +1,20 @@ +{{- range $storage := .Values.storageClass }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ $storage.name }} +provisioner: {{ $.Values.provisionerName }} +allowVolumeExpansion: true +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: {{ default "Delete" $storage.reclaimPolicy }} +parameters: + csi.storage.k8s.io/fstype: {{ default "ext4" $storage.fstype }} + storage: {{ $storage.storage }} + {{- if $storage.cache }} + cache: {{ $storage.cache }} + {{- end }} + {{- if $storage.ssd }} + ssd: "true" + {{- end }} +--- +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.edge.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.edge.yaml new file mode 100644 index 00000000..1b3c37b6 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.edge.yaml @@ -0,0 +1,30 @@ + +controller: + plugin: + image: + pullPolicy: Always + tag: edge + +node: + plugin: + image: + pullPolicy: Always + tag: edge + + nodeSelector: + node.cloudprovider.kubernetes.io/platform: nocloud + +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + ssd: true diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.talos.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.talos.yaml new file mode 100644 index 00000000..76224192 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.talos.yaml @@ -0,0 +1,21 @@ + +node: + nodeSelector: + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.yaml new file mode 100644 index 00000000..1637be82 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.yaml @@ -0,0 +1,222 @@ +# Default values for proxmox-csi-plugin. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +# -- Controller pods priorityClassName. +priorityClassName: system-cluster-critical + +# -- Pods Service Account. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# -- CSI Driver provisioner name. +# Currently, cannot be customized. +provisionerName: csi.proxmox.sinextra.dev + +# -- Cluster name. +# Currently, cannot be customized. +clusterID: kubernetes + +# -- Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md +# for description of individual verbosity levels. +logVerbosityLevel: 5 + +# -- Connection timeout between sidecars. +timeout: 3m + +# -- Proxmox cluster config stored in secrets. +existingConfigSecret: ~ +# -- Proxmox cluster config stored in secrets key. +existingConfigSecretKey: config.yaml + +# -- Proxmox cluster config path. +configFile: /etc/proxmox/config.yaml + +# -- Proxmox cluster config. +config: + clusters: [] + # - url: https://cluster-api-1.exmple.com:8006/api2/json + # insecure: false + # token_id: "login!name" + # token_secret: "secret" + # region: cluster-1 + +# -- Storage class defenition. +storageClass: [] + # - name: proxmox-data-xfs + # storage: data + # reclaimPolicy: Delete + # fstype: ext4|xfs + # + # # https://pve.proxmox.com/wiki/Performance_Tweaks + # cache: directsync|none|writeback|writethrough + # ssd: true + +controller: + plugin: + # -- Controller CSI Driver. + image: + repository: ghcr.io/sergelogvinov/proxmox-csi-controller + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + # -- Controller resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + attacher: + # -- CSI Attacher. + image: + repository: registry.k8s.io/sig-storage/csi-attacher + pullPolicy: IfNotPresent + tag: v4.3.0 + # -- Attacher resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + provisioner: + # -- CSI Provisioner. + image: + repository: registry.k8s.io/sig-storage/csi-provisioner + pullPolicy: IfNotPresent + tag: v3.5.0 + # -- Provisioner resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + resizer: + # -- CSI Resizer. + image: + repository: registry.k8s.io/sig-storage/csi-resizer + pullPolicy: IfNotPresent + tag: v1.8.0 + # -- Resizer resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + +node: + plugin: + # -- Node CSI Driver. + image: + repository: ghcr.io/sergelogvinov/proxmox-csi-node + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + # -- Node CSI Driver resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: {} + driverRegistrar: + # -- Node CSI driver registrar. + image: + repository: registry.k8s.io/sig-storage/csi-node-driver-registrar + pullPolicy: IfNotPresent + tag: v2.8.0 + # -- Node registrar resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + + # -- Node labels for node-plugin assignment. + # ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for node-plugin assignment. + # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: + - key: node.kubernetes.io/unschedulable + operator: Exists + effect: NoSchedule + - key: node.kubernetes.io/disk-pressure + operator: Exists + effect: NoSchedule + +livenessprobe: + # -- Common livenessprobe sidecar. + image: + repository: registry.k8s.io/sig-storage/livenessprobe + pullPolicy: IfNotPresent + tag: v2.10.0 + # -- Failure threshold for livenessProbe + failureThreshold: 5 + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 10 + # -- Period seconds for livenessProbe + periodSeconds: 60 + # -- Liveness probe resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + +# -- Annotations for controller pod. +# ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + +# -- Controller Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + +# -- Controller Container Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + +# -- Controller deployment update stategy type. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +# -- Node labels for controller assignment. +# ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + # node-role.kubernetes.io/control-plane: "" + +# -- Tolerations for controller assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + # - key: node-role.kubernetes.io/control-plane + # effect: NoSchedule + +# -- Affinity for controller assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: {} diff --git a/packages/system/proxmox-csi/patches/namespace.patch b/packages/system/proxmox-csi/patches/namespace.patch new file mode 100644 index 00000000..c0648576 --- /dev/null +++ b/packages/system/proxmox-csi/patches/namespace.patch @@ -0,0 +1,13 @@ +diff --git a/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml b/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml +index 0ed037f..32b065e 100644 +--- a/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml ++++ b/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml +@@ -9,7 +9,7 @@ roleRef: + subjects: + - kind: ServiceAccount + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} +- namespace: kube-system ++ namespace: {{ .Release.Namespace }} + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding diff --git a/packages/system/proxmox-csi/tests/1.yaml b/packages/system/proxmox-csi/tests/1.yaml new file mode 100644 index 00000000..e8a19fc9 --- /dev/null +++ b/packages/system/proxmox-csi/tests/1.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: task-pv-claim +spec: + storageClassName: proxmox-lvm + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 3Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: task-pv-pod +spec: + volumes: + - name: task-pv-storage + persistentVolumeClaim: + claimName: task-pv-claim + containers: + - name: task-pv-container + image: nginx + ports: + - containerPort: 80 + name: "http-server" + volumeMounts: + - mountPath: "/usr/share/nginx/html" + name: task-pv-storage diff --git a/packages/system/proxmox-csi/values.yaml b/packages/system/proxmox-csi/values.yaml new file mode 100644 index 00000000..19a5f9f1 --- /dev/null +++ b/packages/system/proxmox-csi/values.yaml @@ -0,0 +1,22 @@ +proxmox-cloud-controller-manager: + fullnameOverride: proxmox-cloud-controller-manager + + enabledControllers: + - cloud-node + - cloud-node-lifecycle + + # Deploy CCM only on control-plane nodes + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +proxmox-csi-plugin: + fullnameOverride: proxmox-csi-plugin + + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule diff --git a/scripts/common-envs.mk b/scripts/common-envs.mk index 6738677d..222814d7 100644 --- a/scripts/common-envs.mk +++ b/scripts/common-envs.mk @@ -1,4 +1,5 @@ REGISTRY := ghcr.io/aenix-io/cozystack + PUSH := 1 LOAD := 0 VERSION = $(patsubst v%,%,$(shell git describe --tags --abbrev=0)) diff --git a/scripts/installer.sh b/scripts/installer.sh index 5a924002..da3d8fd4 100755 --- a/scripts/installer.sh +++ b/scripts/installer.sh @@ -49,10 +49,10 @@ wait_for_crds() { } install_basic_charts() { - if [ "$BUNDLE" = "paas-full" ] || [ "$BUNDLE" = "distro-full" ]; then - make -C packages/system/cilium apply resume + if [ "$BUNDLE" = "paas-full" ] || [ "$BUNDLE" = "distro-full" || "$BUNDLE" = "paas-proxmox" ]; then + make -C packages/system/cilium apply resume fi - if [ "$BUNDLE" = "paas-full" ]; then + if [ "$BUNDLE" = "paas-full" || "$BUNDLE" = "paas-proxmox" ]; then make -C packages/system/kubeovn apply resume fi }