From 66ce518480118f7ca3f32031644c3b59ee564b0b Mon Sep 17 00:00:00 2001 From: Douglas Mayle Date: Tue, 26 May 2020 18:29:04 +0200 Subject: [PATCH] Add flag for go template to format host names When using an alternate DNS like external-dns, your host names will be different than the internal core-dns format. This adds a flag so that the user can specify the name format using a go template. --- README.md | 1 + docs/custom-name-templates.md | 29 +++++++++++ main.go | 11 +++-- pkg/apis/operator.min.io/v1/helper.go | 49 ++++++++++++++++++- pkg/apis/operator.min.io/v1/helper_test.go | 37 ++++++++++++++ pkg/controller/cluster/csr.go | 7 ++- pkg/controller/cluster/kes-csr.go | 2 +- pkg/controller/cluster/main-controller.go | 13 +++-- .../statefulsets/minio-statefulset.go | 12 +++-- 9 files changed, 144 insertions(+), 17 deletions(-) create mode 100644 docs/custom-name-templates.md diff --git a/README.md b/README.md index 4105639276f..f781102213f 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ MinIO-Operator brings native MinIO, [MCS](https://github.com/minio/mcs), [KES](h | Create and delete highly available distributed MinIO clusters | [Create a MinIO Instance](https://github.com/minio/minio-operator#create-a-minio-instance). | | Automatic TLS for MinIO | [Automatic TLS for MinIO Instance](https://github.com/minio/minio-operator/blob/master/docs/tls.md#automatic-csr-generation). | | Expand an existing MinIO cluster | [Expand a MinIO Cluster](https://github.com/minio/minio-operator/blob/master/docs/adding-zones.md). | +| Use a custom template for hostname discovery | [Custom Hostname Discovery](https://github.com/minio/minio-operator/blob/master/docs/custom-name-templates.md). | | Deploy MCS with MinIO cluster | [Deploy MinIO Instance with MCS](https://github.com/minio/minio-operator/blob/master/docs/mcs.md). | | Deploy KES with MinIO cluster | [Deploy MinIO Instance with KES](https://github.com/minio/minio-operator/blob/master/docs/kes.md). | | Deploy mc mirror | [Deploy Mirror Instance](https://github.com/minio/minio-operator/blob/master/docs/mirror.md). | diff --git a/docs/custom-name-templates.md b/docs/custom-name-templates.md new file mode 100644 index 00000000000..c1921a9d26a --- /dev/null +++ b/docs/custom-name-templates.md @@ -0,0 +1,29 @@ +# Custom Hostname Discovery + +[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) +[![Docker Pulls](https://img.shields.io/docker/pulls/minio/k8s-operator.svg?maxAge=604800)](https://hub.docker.com/r/minio/k8s-operator) + +This document explains how to control the names used for host discovery. This allows us to discover hosts using external name services, which is useful for serving with trusted certificates. + +## Getting Started + +Assuming you have a MinIO cluster with single zone, `zone-0` with 4 drives (as shown in [examples](https://github.com/minio/minio-operator/tree/master/examples)). You can dd a new zone `zone-1` with 4 drives using `kubectl patch` command. + +The example cluster is named minio, so the four servers will be called `minio-0`, `minio-1`, `minio-2`, and `minio-3`. If all of your hosts are available at the domain `example.com` then you can use the `--hosts-template` flag to update discovery: + +``` + containers: + - command: + - /minio-operator + - --hosts-template + - '{{.StatefulSet}}-{{.Ellipsis}}.example.com' +``` + +This will generate the discovery string `minio-{0...3}.example.com`. The following fields are available +| Field | Description | +|-----------------------|-------------| +| StatefulSet | The name of the instance StatefulSet (e.g. `minio`). | +| CIService | The name of the service provided in `spec.serviceName`. | +| HLService | The name of the headless service that is generated (e.g. `minio-hl-service`) | +| Ellipsis | `{0...N-1}` the per-zone host numbers. | +| Domain | The cluster domain, either `cluster.local` or the contents of the `CLUSTER_DOMAIN` environment variable. | diff --git a/main.go b/main.go index fdfe284f3ea..7a41eeaaf10 100644 --- a/main.go +++ b/main.go @@ -45,9 +45,10 @@ import ( var Version = "DEVELOPMENT.GOGET" var ( - masterURL string - kubeconfig string - checkVersion bool + masterURL string + kubeconfig string + hostsTemplate string + checkVersion bool onlyOneSignalHandler = make(chan struct{}) shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} @@ -56,6 +57,7 @@ var ( func init() { flag.StringVar(&kubeconfig, "kubeconfig", "", "path to a kubeconfig. Only required if out-of-cluster") flag.StringVar(&masterURL, "master", "", "the address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster") + flag.StringVar(&hostsTemplate, "hosts-template", "", "the go template to use for hostname formatting of name fields (StatefulSet, CIService, HLService, Ellipsis, Domain)") flag.BoolVar(&checkVersion, "version", false, "print version") } @@ -117,7 +119,8 @@ func main() { kubeInformerFactory.Apps().V1().Deployments(), kubeInformerFactory.Batch().V1().Jobs(), minioInformerFactory.Operator().V1().MinIOInstances(), - kubeInformerFactory.Core().V1().Services()) + kubeInformerFactory.Core().V1().Services(), + hostsTemplate) mirrorController := mirror.NewController(kubeClient, controllerClient, kubeInformerFactory.Batch().V1().Jobs(), diff --git a/pkg/apis/operator.min.io/v1/helper.go b/pkg/apis/operator.min.io/v1/helper.go index 759d9d12b84..27121aa13da 100644 --- a/pkg/apis/operator.min.io/v1/helper.go +++ b/pkg/apis/operator.min.io/v1/helper.go @@ -18,6 +18,7 @@ package v1 import ( + "bytes" "context" "crypto/tls" "errors" @@ -26,6 +27,7 @@ import ( "net/http" "path" "strconv" + "text/template" "time" appsv1 "k8s.io/api/apps/v1" @@ -39,6 +41,19 @@ import ( "github.com/minio/minio/pkg/madmin" ) +type hostsTemplateValues struct { + StatefulSet string + CIService string + HLService string + Ellipsis string + Domain string +} + +// ellipsis returns the host range string +func ellipsis(start, end int) string { + return "{" + strconv.Itoa(start) + "..." + strconv.Itoa(end) + "}" +} + // HasCredsSecret returns true if the user has provided a secret // for a MinIOInstance else false func (mi *MinIOInstance) HasCredsSecret() bool { @@ -77,7 +92,7 @@ func (mi *MinIOInstance) VolumePath() string { if mi.Spec.VolumesPerServer == 1 { return path.Join(mi.Spec.Mountpath, mi.Spec.Subpath) } - return path.Join(mi.Spec.Mountpath+"{0..."+strconv.Itoa((mi.Spec.VolumesPerServer)-1)+"}", mi.Spec.Subpath) + return path.Join(mi.Spec.Mountpath+ellipsis(0, mi.Spec.VolumesPerServer-1), mi.Spec.Subpath) } // MinIOReplicas returns the number of total replicas @@ -203,7 +218,37 @@ func (mi *MinIOInstance) MinIOHosts() []string { // Create the ellipses style URL for _, z := range mi.Spec.Zones { max = max + z.Servers - hosts = append(hosts, fmt.Sprintf("%s-{"+strconv.Itoa(int(index))+"..."+strconv.Itoa(int(max)-1)+"}.%s.%s.svc.%s", mi.MinIOStatefulSetName(), mi.MinIOHLServiceName(), mi.Namespace, ClusterDomain)) + hosts = append(hosts, fmt.Sprintf("%s-%s.%s.%s.svc.%s", ellipsis(int(index), int(max)-1), mi.MinIOStatefulSetName(), mi.MinIOHLServiceName(), mi.Namespace, ClusterDomain)) + index = max + } + return hosts +} + +// TemplatedMinIOHosts returns the domain names in ellipses format created for current MinIOInstance without the service part +func (mi *MinIOInstance) TemplatedMinIOHosts(hostsTemplate string) []string { + hosts := make([]string, 0) + tmpl, err := template.New("hosts").Parse(hostsTemplate) + if err != nil { + msg := "Invalid go template for hosts" + klog.V(2).Infof(msg) + return hosts + } + var max, index int32 + // Create the ellipses style URL + for _, z := range mi.Spec.Zones { + max = max + z.Servers + data := hostsTemplateValues{ + StatefulSet: mi.MinIOStatefulSetName(), + CIService: mi.MinIOCIServiceName(), + HLService: mi.MinIOHLServiceName(), + Ellipsis: ellipsis(int(index), int(max)-1), + Domain: ClusterDomain, + } + output := new(bytes.Buffer) + if err = tmpl.Execute(output, data); err != nil { + continue + } + hosts = append(hosts, output.String()) index = max } return hosts diff --git a/pkg/apis/operator.min.io/v1/helper_test.go b/pkg/apis/operator.min.io/v1/helper_test.go index 47e90473186..0fa820008db 100644 --- a/pkg/apis/operator.min.io/v1/helper_test.go +++ b/pkg/apis/operator.min.io/v1/helper_test.go @@ -5,6 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestEnsureDefaults(t *testing.T) { @@ -42,3 +43,39 @@ func TestEnsureDefaults(t *testing.T) { assert.Equal(t, newImage, mi.Spec.Image) }) } + +func TestTemplateVariables(t *testing.T) { + servers := 2 + mi := MinIOInstance{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: MinIOInstanceSpec{ + Zones: []Zone{{"single", int32(servers)}}, + }, + } + mi.EnsureDefaults() + + t.Run("StatefulSet", func(t *testing.T) { + hosts := mi.TemplatedMinIOHosts("{{.StatefulSet}}") + assert.Contains(t, hosts, mi.MinIOStatefulSetName()) + }) + + t.Run("CIService", func(t *testing.T) { + hosts := mi.TemplatedMinIOHosts("{{.CIService}}") + assert.Contains(t, hosts, mi.MinIOCIServiceName()) + }) + + t.Run("HLService", func(t *testing.T) { + hosts := mi.TemplatedMinIOHosts("{{.HLService}}") + assert.Contains(t, hosts, mi.MinIOHLServiceName()) + }) + + t.Run("Ellipsis", func(t *testing.T) { + hosts := mi.TemplatedMinIOHosts("{{.Ellipsis}}") + assert.Contains(t, hosts, ellipsis(0, servers-1)) + }) + + t.Run("Domain", func(t *testing.T) { + hosts := mi.TemplatedMinIOHosts("{{.Domain}}") + assert.Contains(t, hosts, ClusterDomain) + }) +} diff --git a/pkg/controller/cluster/csr.go b/pkg/controller/cluster/csr.go index 545ed2391a4..a6331e1cbd0 100644 --- a/pkg/controller/cluster/csr.go +++ b/pkg/controller/cluster/csr.go @@ -67,7 +67,7 @@ func isEqual(a, b []string) bool { return true } -func generateCryptoData(mi *miniov1.MinIOInstance) ([]byte, []byte, error) { +func generateCryptoData(mi *miniov1.MinIOInstance, hostsTemplate string) ([]byte, []byte, error) { var dnsNames []string klog.V(0).Infof("Generating private key") privateKey, err := newPrivateKey(miniov1.DefaultEllipticCurve) @@ -85,6 +85,9 @@ func generateCryptoData(mi *miniov1.MinIOInstance) ([]byte, []byte, error) { klog.V(0).Infof("Generating CSR with CN=%s", mi.Spec.CertConfig.CommonName) hosts := mi.AllMinIOHosts() + if hostsTemplate != "" { + hosts = mi.TemplatedMinIOHosts(hostsTemplate) + } if isEqual(mi.Spec.CertConfig.DNSNames, hosts) { dnsNames = mi.Spec.CertConfig.DNSNames @@ -113,7 +116,7 @@ func generateCryptoData(mi *miniov1.MinIOInstance) ([]byte, []byte, error) { // finally creating a secret that MinIO statefulset will use to mount private key and certificate for TLS // This Method Blocks till the CSR Request is approved via kubectl approve func (c *Controller) createCSR(ctx context.Context, mi *miniov1.MinIOInstance) error { - privKeysBytes, csrBytes, err := generateCryptoData(mi) + privKeysBytes, csrBytes, err := generateCryptoData(mi, c.hostsTemplate) if err != nil { klog.Errorf("Private Key and CSR generation failed with error: %v", err) return err diff --git a/pkg/controller/cluster/kes-csr.go b/pkg/controller/cluster/kes-csr.go index f5582d63458..abfba47d26d 100644 --- a/pkg/controller/cluster/kes-csr.go +++ b/pkg/controller/cluster/kes-csr.go @@ -101,7 +101,7 @@ func (c *Controller) createKESTLSCSR(ctx context.Context, mi *miniov1.MinIOInsta // createMinIOClientTLSCSR handles all the steps required to create the CSR: from creation of keys, submitting CSR and // finally creating a secret that KES Statefulset will use for MinIO Client Auth func (c *Controller) createMinIOClientTLSCSR(ctx context.Context, mi *miniov1.MinIOInstance) error { - privKeysBytes, csrBytes, err := generateCryptoData(mi) + privKeysBytes, csrBytes, err := generateCryptoData(mi, c.hostsTemplate) if err != nil { klog.Errorf("Private Key and CSR generation failed with error: %v", err) return err diff --git a/pkg/controller/cluster/main-controller.go b/pkg/controller/cluster/main-controller.go index 05b10cfa294..7206106589a 100644 --- a/pkg/controller/cluster/main-controller.go +++ b/pkg/controller/cluster/main-controller.go @@ -125,6 +125,9 @@ type Controller struct { // recorder is an event recorder for recording Event resources to the // Kubernetes API. recorder record.EventRecorder + + // Use a go template to render the hosts string + hostsTemplate string } // NewController returns a new sample controller @@ -136,7 +139,8 @@ func NewController( deploymentInformer appsinformers.DeploymentInformer, jobInformer batchinformers.JobInformer, minioInstanceInformer informers.MinIOInstanceInformer, - serviceInformer coreinformers.ServiceInformer) *Controller { + serviceInformer coreinformers.ServiceInformer, + hostsTemplate string) *Controller { // Create event broadcaster // Add minio-controller types to the default Kubernetes Scheme so Events can be @@ -164,6 +168,7 @@ func NewController( serviceListerSynced: serviceInformer.Informer().HasSynced, workqueue: queue.NewNamedRateLimitingQueue(queue.DefaultControllerRateLimiter(), "MinIOInstances"), recorder: recorder, + hostsTemplate: hostsTemplate, } klog.Info("Setting up event handlers") @@ -411,7 +416,7 @@ func (c *Controller) syncHandler(key string) error { if err != nil { return err } - ss = statefulsets.NewForMinIO(mi, hlSvc.Name) + ss = statefulsets.NewForMinIO(mi, hlSvc.Name, c.hostsTemplate) ss, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Create(ctx, ss, cOpts) if err != nil { return err @@ -450,7 +455,7 @@ func (c *Controller) syncHandler(key string) error { } } - ss = statefulsets.NewForMinIO(mi, hlSvc.Name) + ss = statefulsets.NewForMinIO(mi, hlSvc.Name, c.hostsTemplate) klog.V(2).Infof("Removing the existing StatefulSet %s with replicas: %d", name, *ss.Spec.Replicas) if err := c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Delete(ctx, ss.Name, metav1.DeleteOptions{}); err != nil { return err @@ -476,7 +481,7 @@ func (c *Controller) syncHandler(key string) error { return err } klog.V(4).Infof("Updating MinIOInstance %s MinIO server version %s, to: %s", name, mi.Spec.Image, ss.Spec.Template.Spec.Containers[0].Image) - ss = statefulsets.NewForMinIO(mi, hlSvc.Name) + ss = statefulsets.NewForMinIO(mi, hlSvc.Name, c.hostsTemplate) if _, err := c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Update(ctx, ss, uOpts); err != nil { return err } diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index 9e0c22287f1..eea22fc4143 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -183,7 +183,7 @@ func probes(mi *miniov1.MinIOInstance) (readiness, liveness *corev1.Probe) { } // Builds the MinIO container for a MinIOInstance. -func minioServerContainer(mi *miniov1.MinIOInstance, serviceName string) corev1.Container { +func minioServerContainer(mi *miniov1.MinIOInstance, serviceName string, hostsTemplate string) corev1.Container { args := []string{"server", "--certs-dir", "/tmp/certs"} if mi.Spec.Zones[0].Servers == 1 { @@ -191,7 +191,11 @@ func minioServerContainer(mi *miniov1.MinIOInstance, serviceName string) corev1. args = append(args, mi.VolumePath()) } else { // append all the MinIOInstance replica URLs - for _, h := range mi.MinIOHosts() { + hosts := mi.MinIOHosts() + if hostsTemplate != "" { + hosts = mi.TemplatedMinIOHosts(hostsTemplate) + } + for _, h := range hosts { args = append(args, fmt.Sprintf("%s://"+h+"%s", miniov1.Scheme, mi.VolumePath())) } } @@ -245,7 +249,7 @@ func getVolumesForContainer(mi *miniov1.MinIOInstance) []corev1.Volume { } // NewForMinIO creates a new StatefulSet for the given Cluster. -func NewForMinIO(mi *miniov1.MinIOInstance, serviceName string) *appsv1.StatefulSet { +func NewForMinIO(mi *miniov1.MinIOInstance, serviceName string, hostsTemplate string) *appsv1.StatefulSet { // If a PV isn't specified just use a EmptyDir volume var podVolumes = getVolumesForContainer(mi) var replicas = mi.MinIOReplicas() @@ -327,7 +331,7 @@ func NewForMinIO(mi *miniov1.MinIOInstance, serviceName string) *appsv1.Stateful }) } - containers := []corev1.Container{minioServerContainer(mi, serviceName)} + containers := []corev1.Container{minioServerContainer(mi, serviceName, hostsTemplate)} ss := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{