local pv provisioner for public cloud

Makefile

@@ -75,6 +75,7 @@ build: ## Build binary.
  $(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/controller-manager cmd/controller-manager/main.go
  $(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/autoscaler cmd/autoscaler/main.go
  $(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/scheduler cmd/scheduler/main.go
+ $(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/provisioner cmd/provisioner/main.go
 
 helm-charts: ## Build helm charts.
  helm package charts/nebula-operator --version $(CHARTS_VERSION) --app-version $(CHARTS_VERSION)

alpine.multiarch

@@ -1,9 +1,9 @@
 FROM alpine:3.18.2
 
-ARG TARGETPLATFORM
+ARG TARGETARCH
 
 RUN apk update \
  && apk upgrade \
  && apk add --no-cache \
- curl jq \
+ curl jq util-linux bash xfsprogs \
  && rm -rf /var/cache/apk/*

cmd/provisioner/app/options/options.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2023 Vesoft Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+ "time"
+
+ "github.com/spf13/pflag"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ pvController "sigs.k8s.io/sig-storage-lib-external-provisioner/v9/controller"
+)
+
+const (
+ defaultProvisioner = "nebula-cloud.io/local-pv"
+ defaultHelperImage = "vesoft/nebula-alpine:latest"
+ defaultServiceAccount = "local-pv-provisioner-sa"
+ defaultConfigMap = "local-pv-config"
+
+ defaultResyncPeriod = 5 * time.Minute
+ defaultThreadiness = pvController.DefaultThreadiness
+ defaultFailedProvisionThreshold = pvController.DefaultFailedProvisionThreshold
+ defaultFailedDeleteThreshold = pvController.DefaultFailedDeleteThreshold
+)
+
+type Options struct {
+ // leaderElect enables a leader election client to gain leadership
+ // before executing the main loop
+ LeaderElect bool
+
+ // The name of the provisioner for which this controller dynamically
+ // provisions volumes
+ Provisioner string
+
+ // The helper image used for managing mount point on the host
+ HelperImage string
+
+ // Image pull secret used for pulling helper image
+ ImagePullSecret string
+
+ // The name of the provisioner service account
+ ServiceAccount string
+
+ // The name of the provisioner configmap
+ ConfigMap string
+
+ // The number of claim and volume workers each to launch
+ Threadiness int
+
+ // The threshold for max number of retries on failures of volume Provision
+ FailedProvisionThreshold int
+
+ // The threshold for max number of retries on failures of volume Delete
+ FailedDeleteThreshold int
+
+ // Period at which the controller forces the repopulation of its local object stores
+ ResyncPeriod metav1.Duration
+}
+
+func NewOptions() *Options {
+ return &Options{}
+}
+
+func (o *Options) AddFlags(flags *pflag.FlagSet) {
+ flags.BoolVar(&o.LeaderElect, "leader-elect", false, "Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.")
+ flags.StringVar(&o.Provisioner, "provisioner-name", defaultProvisioner, "The name of the provisioner for which this controller dynamically provisions volumes.")
+ flags.StringVar(&o.HelperImage, "helper-image", defaultHelperImage, "The helper image used for managing mount point on the host.")
+ flags.StringVar(&o.ImagePullSecret, "image-pull-secret", "", "Image pull secret used for pulling helper image.")
+ flags.StringVar(&o.ServiceAccount, "service-account", defaultServiceAccount, "The name of the provisioner service account.")
+ flags.StringVar(&o.ConfigMap, "configmap", defaultConfigMap, "The name of the provisioner configmap.")
+ flags.IntVar(&o.Threadiness, "threadiness", defaultThreadiness, "The number of claim and volume workers each to launch.")
+ flags.IntVar(&o.FailedProvisionThreshold, "failed-provision-threshold", defaultFailedProvisionThreshold, "The threshold for max number of retries on failures of volume Provision.")
+ flags.IntVar(&o.FailedDeleteThreshold, "failed-delete-threshold", defaultFailedDeleteThreshold, "The threshold for max number of retries on failures of volume Delete.")
+ flags.DurationVar(&o.ResyncPeriod.Duration, "resync-period", defaultResyncPeriod, "Period at which the controller forces the repopulation of its local object stores.")
+}

cmd/provisioner/app/provisioner.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2023 Vesoft Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+ "context"
+ "flag"
+ "os"
+ "time"
+
+ "github.com/spf13/cobra"
+ "k8s.io/client-go/informers"
+ clientset "k8s.io/client-go/kubernetes"
+ cliflag "k8s.io/component-base/cli/flag"
+ "k8s.io/klog/v2"
+ "sigs.k8s.io/controller-runtime/pkg/client/config"
+ pvController "sigs.k8s.io/sig-storage-lib-external-provisioner/v9/controller"
+
+ "github.com/vesoft-inc/nebula-operator/cmd/provisioner/app/options"
+ "github.com/vesoft-inc/nebula-operator/pkg/controller/provisioner"
+ klogflag "github.com/vesoft-inc/nebula-operator/pkg/flag/klog"
+)
+
+const (
+ defaultProvisionerNamespace = "local-pv-storage"
+)
+
+// NewProvisionerCommand creates a *cobra.Command object with default parameters
+func NewProvisionerCommand(ctx context.Context) *cobra.Command {
+ opts := options.NewOptions()
+
+ cmd := &cobra.Command{
+ Use: "local-pv-provisioner",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return Run(ctx, opts)
+ },
+ }
+
+ nfs := cliflag.NamedFlagSets{}
+ fs := nfs.FlagSet("generic")
+ fs.AddGoFlagSet(flag.CommandLine)
+ opts.AddFlags(fs)
+
+ logsFlagSet := nfs.FlagSet("logs")
+ klogflag.Add(logsFlagSet)
+
+ cmd.Flags().AddFlagSet(fs)
+ cmd.Flags().AddFlagSet(logsFlagSet)
+
+ return cmd
+}
+
+// Run runs the provisioner with options. This should never exit.
+func Run(ctx context.Context, opts *options.Options) error {
+ c, err := config.GetConfig()
+ if err != nil {
+ return err
+ }
+ kubeClient, err := clientset.NewForConfig(c)
+ if err != nil {
+ return err
+ }
+
+ cacheConfig := &provisioner.CacheConfig{
+ ProvisionerName: opts.Provisioner,
+ Cache: provisioner.NewVolumeCache(),
+ InformerFactory: informers.NewSharedInformerFactory(kubeClient, time.Second*5),
+ }
+ provisioner.NewPopulator(cacheConfig)
+
+ // Start informers after all event listeners are registered.
+ cacheConfig.InformerFactory.Start(ctx.Done())
+ // Wait for all started informers' cache were synced.
+ for v, synced := range cacheConfig.InformerFactory.WaitForCacheSync(ctx.Done()) {
+ if !synced {
+ klog.Fatalf("Error syncing informer for %v", v)
+ }
+ }
+
+ namespace := os.Getenv("POD_NAMESPACE")
+ if namespace == "" {
+ namespace = defaultProvisionerNamespace
+ }
+ p := provisioner.NewProvisioner(ctx, kubeClient, namespace, opts.HelperImage, opts.ConfigMap, opts.ServiceAccount)
+ p.CacheConfig = cacheConfig
+ if opts.ImagePullSecret != "" {
+ p.SetImagePullSecret(opts.ImagePullSecret)
+ }
+ pc := pvController.NewProvisionController(
+ kubeClient,
+ opts.Provisioner,
+ p,
+ pvController.LeaderElection(opts.LeaderElect),
+ pvController.FailedProvisionThreshold(opts.FailedProvisionThreshold),
+ pvController.FailedDeleteThreshold(opts.FailedDeleteThreshold),
+ pvController.Threadiness(opts.Threadiness),
+ )
+ klog.Info("local pv provisioner started")
+ pc.Run(ctx)
+ klog.Info("local pv provisioner stopped")
+
+ return nil
+}

cmd/provisioner/main.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2023 Vesoft Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "os"
+
+ "k8s.io/component-base/cli"
+ ctrl "sigs.k8s.io/controller-runtime"
+
+ "github.com/vesoft-inc/nebula-operator/cmd/provisioner/app"
+)
+
+func main() {
+ ctx := ctrl.SetupSignalHandler()
+ cmd := app.NewProvisionerCommand(ctx)
+ code := cli.Run(cmd)
+ os.Exit(code)
+}

config/samples/gke-daemonset-raid-disks.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: gke-raid-disks
+ namespace: default
+ labels:
+ k8s-app: gke-raid-disks
+spec:
+ selector:
+ matchLabels:
+ name: gke-raid-disks
+ template:
+ metadata:
+ labels:
+ name: gke-raid-disks
+ spec:
+ nodeSelector:
+ cloud.google.com/gke-local-nvme-ssd: "true"
+ hostPID: true
+ containers:
+ - name: startup-script
+ image: gcr.io/google-containers/startup-script:v1
+ securityContext:
+ privileged: true
+ env:
+ - name: STARTUP_SCRIPT
+ value: |
+ set -o errexit
+ set -o nounset
+ set -o pipefail
+
+ devices=()
+ for ssd in /dev/disk/by-id/google-local-ssd-block*; do
+ if [ -e "${ssd}" ]; then
+ devices+=("${ssd}")
+ fi
+ done
+ if [ "${#devices[@]}" -eq 0 ]; then
+ echo "No Local NVMe SSD disks found."
+ exit 0
+ fi
+
+ seen_arrays=(/dev/md/*)
+ device=${seen_arrays[0]}
+ echo "Setting RAID array with Local SSDs on device ${device}"
+ if [ ! -e "$device" ]; then
+ device="/dev/md/0"
+ echo "y" | mdadm --create "${device}" --level=0 --force --raid-devices=${#devices[@]} "${devices[@]}"
+ fi
+
+ if ! tune2fs -l "${device}" ; then
+ echo "Formatting '${device}'"
+ mkfs.ext4 -F "${device}"
+ fi
+
+ mountpoint=/mnt/disks/raid0
+ mkdir -p "${mountpoint}"
+ echo "Mounting '${device}' at '${mountpoint}'"
+ mount -o discard,defaults "${device}" "${mountpoint}"
+ chmod a+w "${mountpoint}"