From 024937fa989de183d9999dc8749daed3f8e0d28a Mon Sep 17 00:00:00 2001 From: Douglas Mayle Date: Tue, 26 May 2020 18:29:04 +0200 Subject: [PATCH] Add flag for go template to format host names When using an alternate DNS like external-dns, your host names will be different than the internal core-dns format. This adds a flag so that the user can specify the name format using a go template. --- README.md | 1 + docs/custom-name-templates.md | 25 ++++++++++++ main.go | 11 +++-- pkg/apis/operator.min.io/v1/helper.go | 40 +++++++++++++++++++ pkg/controller/cluster/main-controller.go | 13 ++++-- .../statefulsets/minio-statefulset.go | 12 ++++-- 6 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 docs/custom-name-templates.md diff --git a/README.md b/README.md index 4105639276f..f781102213f 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ MinIO-Operator brings native MinIO, [MCS](https://github.com/minio/mcs), [KES](h | Create and delete highly available distributed MinIO clusters | [Create a MinIO Instance](https://github.com/minio/minio-operator#create-a-minio-instance). | | Automatic TLS for MinIO | [Automatic TLS for MinIO Instance](https://github.com/minio/minio-operator/blob/master/docs/tls.md#automatic-csr-generation). | | Expand an existing MinIO cluster | [Expand a MinIO Cluster](https://github.com/minio/minio-operator/blob/master/docs/adding-zones.md). | +| Use a custom template for hostname discovery | [Custom Hostname Discovery](https://github.com/minio/minio-operator/blob/master/docs/custom-name-templates.md). | | Deploy MCS with MinIO cluster | [Deploy MinIO Instance with MCS](https://github.com/minio/minio-operator/blob/master/docs/mcs.md). | | Deploy KES with MinIO cluster | [Deploy MinIO Instance with KES](https://github.com/minio/minio-operator/blob/master/docs/kes.md). | | Deploy mc mirror | [Deploy Mirror Instance](https://github.com/minio/minio-operator/blob/master/docs/mirror.md). | diff --git a/docs/custom-name-templates.md b/docs/custom-name-templates.md new file mode 100644 index 00000000000..136f1e8a238 --- /dev/null +++ b/docs/custom-name-templates.md @@ -0,0 +1,25 @@ +# Custom Hostname Discovery + +[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) +[![Docker Pulls](https://img.shields.io/docker/pulls/minio/k8s-operator.svg?maxAge=604800)](https://hub.docker.com/r/minio/k8s-operator) + +This document explains how to control the names used for host discovery. This allows us to discover hosts using external name services, which is useful for serving with trusted certificates. + +## Getting Started + +Assuming you have a MinIO cluster with single zone, `zone-0` with 4 drives (as shown in [examples](https://github.com/minio/minio-operator/tree/master/examples)). You can dd a new zone `zone-1` with 4 drives using `kubectl patch` command. + +The example cluster is named minio, so the four servers will be called `minio-0`, `minio-1`, `minio-2`, and `minio-3`. If all of your hosts are available at the domain `example.com` then you can use the `--hosts-template` flag to update discovery: + +``` +/minio-operator --hosts-template "{{.StatefulSet}}-{{.Ellipsis}}.example.com" +``` + +This will generate the discovery string `minio-{0...3}.example.com`. The following fields are available +| Field | Description | +|-----------------------|-------------| +| StatefulSet | The name of the instance StatefulSet (e.g. `minio`). | +| CIService | The name of the service provided in `spec.serviceName`. | +| HLService | The name of the headless service that is generated (e.g. `minio-hl-service`) | +| Ellipsis | `{0...N-1}` the per-zone host numbers. | +| Domain | The cluster domain, either `cluster.local` or the contents of the `CLUSTER_DOMAIN` environment variable. | diff --git a/main.go b/main.go index fdfe284f3ea..7a41eeaaf10 100644 --- a/main.go +++ b/main.go @@ -45,9 +45,10 @@ import ( var Version = "DEVELOPMENT.GOGET" var ( - masterURL string - kubeconfig string - checkVersion bool + masterURL string + kubeconfig string + hostsTemplate string + checkVersion bool onlyOneSignalHandler = make(chan struct{}) shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} @@ -56,6 +57,7 @@ var ( func init() { flag.StringVar(&kubeconfig, "kubeconfig", "", "path to a kubeconfig. Only required if out-of-cluster") flag.StringVar(&masterURL, "master", "", "the address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster") + flag.StringVar(&hostsTemplate, "hosts-template", "", "the go template to use for hostname formatting of name fields (StatefulSet, CIService, HLService, Ellipsis, Domain)") flag.BoolVar(&checkVersion, "version", false, "print version") } @@ -117,7 +119,8 @@ func main() { kubeInformerFactory.Apps().V1().Deployments(), kubeInformerFactory.Batch().V1().Jobs(), minioInformerFactory.Operator().V1().MinIOInstances(), - kubeInformerFactory.Core().V1().Services()) + kubeInformerFactory.Core().V1().Services(), + hostsTemplate) mirrorController := mirror.NewController(kubeClient, controllerClient, kubeInformerFactory.Batch().V1().Jobs(), diff --git a/pkg/apis/operator.min.io/v1/helper.go b/pkg/apis/operator.min.io/v1/helper.go index f3712fe1117..41e77041bc5 100644 --- a/pkg/apis/operator.min.io/v1/helper.go +++ b/pkg/apis/operator.min.io/v1/helper.go @@ -18,6 +18,7 @@ package v1 import ( + "bytes" "context" "crypto/tls" "errors" @@ -26,6 +27,7 @@ import ( "net/http" "path" "strconv" + "text/template" "time" appsv1 "k8s.io/api/apps/v1" @@ -39,6 +41,14 @@ import ( "github.com/minio/minio/pkg/madmin" ) +type hostsTemplateValues struct { + StatefulSet string + CIService string + HLService string + Ellipsis string + Domain string +} + // HasCredsSecret returns true if the user has provided a secret // for a MinIOInstance else false func (mi *MinIOInstance) HasCredsSecret() bool { @@ -221,6 +231,36 @@ func (mi *MinIOInstance) MinIOHosts() []string { return hosts } +// TemplatedMinIOHosts returns the domain names in ellipses format created for current MinIOInstance without the service part +func (mi *MinIOInstance) TemplatedMinIOHosts(hostsTemplate string) []string { + hosts := make([]string, 0) + tmpl, err := template.New("hosts").Parse(hostsTemplate) + if err != nil { + msg := "Invalid go template for hosts" + klog.V(2).Infof(msg) + return hosts + } + var max, index int32 + // Create the ellipses style URL + for _, z := range mi.Spec.Zones { + max = max + z.Servers + data := hostsTemplateValues{ + StatefulSet: mi.MinIOStatefulSetName(), + CIService: mi.MinIOCIServiceName(), + HLService: mi.MinIOHLServiceName(), + Ellipsis: "{" + strconv.Itoa(int(index)) + "..." + strconv.Itoa(int(max)-1) + "}", + Domain: ClusterDomain, + } + output := new(bytes.Buffer) + if err = tmpl.Execute(output, data); err != nil { + continue + } + hosts = append(hosts, output.String()) + index = max + } + return hosts +} + // AllMinIOHosts returns the all the individual domain names relevant for current MinIOInstance func (mi *MinIOInstance) AllMinIOHosts() []string { hosts := make([]string, 0) diff --git a/pkg/controller/cluster/main-controller.go b/pkg/controller/cluster/main-controller.go index e6d455a7aa9..41e295787be 100644 --- a/pkg/controller/cluster/main-controller.go +++ b/pkg/controller/cluster/main-controller.go @@ -125,6 +125,9 @@ type Controller struct { // recorder is an event recorder for recording Event resources to the // Kubernetes API. recorder record.EventRecorder + + // Use a go template to render the hosts string + hostsTemplate string } // NewController returns a new sample controller @@ -136,7 +139,8 @@ func NewController( deploymentInformer appsinformers.DeploymentInformer, jobInformer batchinformers.JobInformer, minioInstanceInformer informers.MinIOInstanceInformer, - serviceInformer coreinformers.ServiceInformer) *Controller { + serviceInformer coreinformers.ServiceInformer, + hostsTemplate string) *Controller { // Create event broadcaster // Add minio-controller types to the default Kubernetes Scheme so Events can be @@ -164,6 +168,7 @@ func NewController( serviceListerSynced: serviceInformer.Informer().HasSynced, workqueue: queue.NewNamedRateLimitingQueue(queue.DefaultControllerRateLimiter(), "MinIOInstances"), recorder: recorder, + hostsTemplate: hostsTemplate, } klog.Info("Setting up event handlers") @@ -411,7 +416,7 @@ func (c *Controller) syncHandler(key string) error { if err != nil { return err } - ss = statefulsets.NewForMinIO(mi, hlSvc.Name) + ss = statefulsets.NewForMinIO(mi, hlSvc.Name, c.hostsTemplate) ss, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Create(ctx, ss, cOpts) if err != nil { return err @@ -446,7 +451,7 @@ func (c *Controller) syncHandler(key string) error { } } - ss = statefulsets.NewForMinIO(mi, hlSvc.Name) + ss = statefulsets.NewForMinIO(mi, hlSvc.Name, c.hostsTemplate) klog.V(2).Infof("Removing the existing StatefulSet %s with replicas: %d", name, *ss.Spec.Replicas) if err := c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Delete(ctx, ss.Name, metav1.DeleteOptions{}); err != nil { return err @@ -472,7 +477,7 @@ func (c *Controller) syncHandler(key string) error { return err } klog.V(4).Infof("Updating MinIOInstance %s MinIO server version %s, to: %s", name, mi.Spec.Image, ss.Spec.Template.Spec.Containers[0].Image) - ss = statefulsets.NewForMinIO(mi, hlSvc.Name) + ss = statefulsets.NewForMinIO(mi, hlSvc.Name, c.hostsTemplate) if _, err := c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Update(ctx, ss, uOpts); err != nil { return err } diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index 9bfcf751c1e..b75bfda489b 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -186,7 +186,7 @@ func probes(mi *miniov1.MinIOInstance) (readiness, liveness *corev1.Probe) { } // Builds the MinIO container for a MinIOInstance. -func minioServerContainer(mi *miniov1.MinIOInstance, serviceName string) corev1.Container { +func minioServerContainer(mi *miniov1.MinIOInstance, serviceName string, hostsTemplate string) corev1.Container { args := []string{"server", "--certs-dir", "/tmp/certs"} if mi.Spec.Zones[0].Servers == 1 { @@ -194,7 +194,11 @@ func minioServerContainer(mi *miniov1.MinIOInstance, serviceName string) corev1. args = append(args, miniov1.MinIOVolumeMountPath) } else { // append all the MinIOInstance replica URLs - for _, h := range mi.MinIOHosts() { + hosts := mi.MinIOHosts() + if hostsTemplate != "" { + hosts = mi.TemplatedMinIOHosts(hostsTemplate) + } + for _, h := range hosts { args = append(args, fmt.Sprintf("%s://"+h+"%s", miniov1.Scheme, mi.VolumePath())) } } @@ -248,7 +252,7 @@ func getVolumesForContainer(mi *miniov1.MinIOInstance) []corev1.Volume { } // NewForMinIO creates a new StatefulSet for the given Cluster. -func NewForMinIO(mi *miniov1.MinIOInstance, serviceName string) *appsv1.StatefulSet { +func NewForMinIO(mi *miniov1.MinIOInstance, serviceName string, hostsTemplate string) *appsv1.StatefulSet { // If a PV isn't specified just use a EmptyDir volume var podVolumes = getVolumesForContainer(mi) var replicas = mi.MinIOReplicas() @@ -330,7 +334,7 @@ func NewForMinIO(mi *miniov1.MinIOInstance, serviceName string) *appsv1.Stateful }) } - containers := []corev1.Container{minioServerContainer(mi, serviceName)} + containers := []corev1.Container{minioServerContainer(mi, serviceName, hostsTemplate)} ss := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{