Add support for --volumes-per-server flag in the kubectl minio plugin…

… during creation or expansion of a tenant

Allow disable antiaffinity with expansion of a tenant

go.mod

@@ -60,6 +60,7 @@ require (
 require (
 	github.com/go-test/deep v1.1.0
 	github.com/minio/kes-go v0.2.1
+	github.com/minio/madmin-go/v2 v2.2.1
 	golang.org/x/mod v0.14.0
 	sigs.k8s.io/controller-runtime v0.16.3
 )

go.sum

@@ -200,16 +200,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A=
 github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
-github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80=
-github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
 github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
 github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
 github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
 github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
 github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
 github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
-github.com/lestrrat-go/jwx v1.2.26 h1:4iFo8FPRZGDYe1t19mQP0zTRqA7n8HnJ5lkIiDvJcB0=
-github.com/lestrrat-go/jwx v1.2.26/go.mod h1:MaiCdGbn3/cckbOFSCluJlJMmp9dmZm5hDuIkx8ftpQ=
 github.com/lestrrat-go/jwx v1.2.27 h1:cvnTnda/YzdyFuWdEAMkI6BsLtItSrASEVCI3C/IUEQ=
 github.com/lestrrat-go/jwx v1.2.27/go.mod h1:Stob9LjSqR3lOmNdxF0/TvZo60V3hUGv8Fr7Bwzla3k=
 github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
@@ -258,6 +254,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/minio/kes-go v0.2.1 h1:KnqS+p6xoSFJZbQhmJaz/PbxeA6nQyRqT/ywrn5lU2o=
 github.com/minio/kes-go v0.2.1/go.mod h1:76xf7l41Wrh+IifisABXK2S8uZWYgWV1IGBKC3GdOJk=
+github.com/minio/madmin-go/v2 v2.2.1 h1:zyRcXBm013VF6+7wefOpJEU6N2f7/7uFKsOrcn44DpM=
+github.com/minio/madmin-go/v2 v2.2.1/go.mod h1:8bL1RMNkblIENFSgGYjeHrzUx9PxROb7OqfNuMU9ivE=
 github.com/minio/madmin-go/v3 v3.0.35 h1:cCo5ZZpHA+rlBQbsAcwFwiuh/uHJmjVoDDx1G4+zaho=
 github.com/minio/madmin-go/v3 v3.0.35/go.mod h1:4QN2NftLSV7MdlT50dkrenOMmNVHluxTvlqJou3hte8=
 github.com/minio/mc v0.0.0-20231202112410-d920e2b34b22 h1:U3IKVR1Bi3MqnfLTZw2JUlrA2P1aQGSDESDyYUXqPtI=
@@ -442,7 +440,6 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
 golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
 golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=

kubectl-minio/cmd/helpers/helpers.go

@@ -184,7 +184,11 @@ func CapacityPerVolume(capacity string, volumes int32) (*resource.Quantity, erro
 	if err != nil {
 		return nil, err
 	}
-	return resource.NewQuantity(totalQuantity.Value()/int64(volumes), totalQuantity.Format), nil
+	quantity := resource.NewQuantity(totalQuantity.Value()/int64(volumes), totalQuantity.Format)
+	if quantity.Sign() <= 0 {
+		return nil, errors.New("capacity per volume needs to be greater than zero")
+	}
+	return quantity, nil
 }
 
 // TotalCapacity returns total capacity of a given tenant

kubectl-minio/cmd/resources/tenant.go

@@ -17,7 +17,6 @@ package resources
 
 import (
 	"errors"
-
 	"github.com/minio/kubectl-minio/cmd/helpers"
 	operator "github.com/minio/operator/pkg/apis/minio.min.io"
 	miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2"
@@ -33,6 +32,7 @@ type TenantOptions struct {
 	ConfigurationSecretName string
 	Servers                 int32
 	Volumes                 int32
+	VolumesPerServer        int32
 	Capacity                string
 	NS                      string
 	Image                   string
@@ -55,22 +55,35 @@ func (t TenantOptions) Validate() error {
 	if t.Servers <= 0 {
 		return errors.New("--servers is required. Specify a value greater than or equal to 1")
 	}
-	if t.Volumes <= 0 {
-		return errors.New("--volumes is required. Specify a positive value")
+	if t.Volumes <= 0 && t.VolumesPerServer <= 0 {
+		return errors.New("--volumes or --volumes-per-server is required. Specify either with a value greater than or equal to 1")
+	}
+	if t.Volumes > 0 && t.VolumesPerServer > 0 {
+		return errors.New("only either --volumes or --volumes-per-server may be specified")
+	}
+	if t.VolumesPerServer > 0 {
+		t.Volumes = t.VolumesPerServer * t.Servers
 	}
 	if t.Capacity == "" {
 		return errors.New("--capacity flag is required")
 	}
-	_, err := resource.ParseQuantity(t.Capacity)
+	capacity, err := resource.ParseQuantity(t.Capacity)
 	if err != nil {
 		if err == resource.ErrFormatWrong {
-			return errors.New("--capacity flag is incorrectly formatted. Please use suffix like 'T' or 'Ti' only")
+			return errors.New("--capacity flag is incorrectly formatted. Use a suffix like 'T' or 'Ti' only")
 		}
 		return err
 	}
+	if capacity.Sign() <= 0 {
+		return errors.New("--capacity needs to be greater than zero")
+	}
 	if t.Volumes%t.Servers != 0 {
 		return errors.New("--volumes should be a multiple of --servers")
 	}
+	_, err = helpers.CapacityPerVolume(t.Capacity, t.Volumes)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -104,7 +117,13 @@ func storageClass(sc string) *string {
 // NewTenant will return a new Tenant for a MinIO Operator
 func NewTenant(opts *TenantOptions, userSecret *v1.Secret) (*miniov2.Tenant, error) {
 	autoCert := !opts.DisableTLS
-	volumesPerServer := helpers.VolumesPerServer(opts.Volumes, opts.Servers)
+	// Derive Volumes or VolumesPerServer in the absence of the other
+	// Exclusively either variable is guaranteed to exist
+	if opts.Volumes == 0 {
+		opts.Volumes = opts.VolumesPerServer * opts.Servers
+	} else {
+		opts.VolumesPerServer = helpers.VolumesPerServer(opts.Volumes, opts.Servers)
+	}
 	capacityPerVolume, err := helpers.CapacityPerVolume(opts.Capacity, opts.Volumes)
 	if err != nil {
 		return nil, err
@@ -128,7 +147,7 @@ func NewTenant(opts *TenantOptions, userSecret *v1.Secret) (*miniov2.Tenant, err
 				Console: opts.ExposeConsoleService,
 				MinIO:   opts.ExposeMinioService,
 			},
-			Pools:           []miniov2.Pool{Pool(opts, volumesPerServer, *capacityPerVolume)},
+			Pools:           []miniov2.Pool{Pool(opts, opts.VolumesPerServer, *capacityPerVolume)},
 			RequestAutoCert: &autoCert,
 			Mountpath:       helpers.MinIOMountPath,
 			KES:             tenantKESConfig(opts.Name, opts.KmsSecret, opts.KesImage),

kubectl-minio/cmd/tenant-create.go

@@ -55,12 +55,12 @@ func newTenantCreateCmd(out io.Writer, errOut io.Writer) *cobra.Command {
 	c := &createCmd{out: out, errOut: errOut}
 
 	cmd := &cobra.Command{
-		Use:     "create <TENANTNAME> --pool <POOLNAME> --servers <NSERVERS> --volumes <NVOLUMES> --capacity <SIZE> --namespace <TENANTNS>",
+		Use:     "create <TENANTNAME> --pool <POOLNAME> --servers <NSERVERS> ( --volumes <NVOLUMES> | --volumes-per-server <NVOLUMESPERSERVER> ) --capacity <SIZE> --namespace <TENANTNS>",
 		Short:   "Create a MinIO tenant",
 		Long:    createDesc,
 		Example: createExample,
 		Args: func(cmd *cobra.Command, args []string) error {
-			// The disable-tls parameter default value is false, we cannot rely on the default value binded to the tenantOpts.DisableTLS variable
+			// The disable-tls parameter default value is false, we cannot rely on the default value bound to the tenantOpts.DisableTLS variable
 			// to identify if the parameter --disable-tls was actually set on the command line.
 			// regardless of which value is being set to the flag, if the flag and ONLY if the flag is present, then we disable TLS
 			c.tenantOpts.DisableTLS = cmd.Flags().Lookup("disable-tls").Changed
@@ -80,6 +80,7 @@ func newTenantCreateCmd(out io.Writer, errOut io.Writer) *cobra.Command {
 	f.StringVarP(&c.tenantOpts.PoolName, "pool", "p", "", "name for this pool")
 	f.Int32Var(&c.tenantOpts.Servers, "servers", 0, "total number of pods in MinIO tenant")
 	f.Int32Var(&c.tenantOpts.Volumes, "volumes", 0, "total number of volumes in the MinIO tenant")
+	f.Int32Var(&c.tenantOpts.VolumesPerServer, "volumes-per-server", 0, "number of volumes in each server in the MinIO tenant")
 	f.StringVar(&c.tenantOpts.Capacity, "capacity", "", "total raw capacity of MinIO tenant in this pool, e.g. 16Ti")
 	f.StringVarP(&c.tenantOpts.NS, "namespace", "n", "", "k8s namespace for this MinIO tenant")
 	f.StringVarP(&c.tenantOpts.StorageClass, "storage-class", "s", helpers.DefaultStorageclass, "storage class for this MinIO tenant")
@@ -182,7 +183,11 @@ func (c *createCmd) populateInteractiveTenant() error {
 	c.tenantOpts.Name = helpers.AskQuestion("Tenant name", helpers.CheckValidTenantName)
 	c.tenantOpts.ConfigurationSecretName = fmt.Sprintf("%s-env-configuration", c.tenantOpts.Name)
 	c.tenantOpts.Servers = int32(helpers.AskNumber("Total of servers", greaterThanZero))
-	c.tenantOpts.Volumes = int32(helpers.AskNumber("Total of volumes", greaterThanZero))
+	if helpers.Ask("Define 'Total of volumes'") {
+		c.tenantOpts.Volumes = int32(helpers.AskNumber("Total of volumes", greaterThanZero))
+	} else {
+		c.tenantOpts.VolumesPerServer = int32(helpers.AskNumber("Volumes per server", greaterThanZero))
+	}
 	c.tenantOpts.NS = helpers.AskQuestion("Namespace", validateEmptyInput)
 	c.tenantOpts.Capacity = helpers.AskQuestion("Capacity", validateCapacity)
 	if err := c.tenantOpts.Validate(); err != nil {
@@ -192,6 +197,7 @@ func (c *createCmd) populateInteractiveTenant() error {
 	c.tenantOpts.ExposeMinioService = helpers.Ask("Expose Minio Service")
 	c.tenantOpts.ExposeConsoleService = helpers.Ask("Expose Console Service")
 	c.tenantOpts.EnableSFTP = helpers.Ask("Enable SFTP")
+	c.tenantOpts.DisableAntiAffinity = helpers.Ask("Disable Anti-Affinity (unsupported in production environment)")
 	return nil
 }
 
@@ -206,7 +212,16 @@ func validateCapacity(value string) error {
 	if err := validateEmptyInput(value); err != nil {
 		return err
 	}
-	_, err := resource.ParseQuantity(value)
+	capacity, err := resource.ParseQuantity(value)
+	if err != nil {
+		if err == resource.ErrFormatWrong {
+			return errors.New("capacity flag is incorrectly formatted. Use a suffix like 'T' or 'Ti' only")
+		}
+		return err
+	}
+	if capacity.Sign() <= 0 {
+		return errors.New("capacity needs to be greater than zero")
+	}
 	return err
 }
 

kubectl-minio/cmd/tenant-expand.go

@@ -20,10 +20,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
-
 	"github.com/minio/kubectl-minio/cmd/helpers"
 	"github.com/minio/kubectl-minio/cmd/resources"
+	"io"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
@@ -51,7 +50,7 @@ func newTenantExpandCmd(out io.Writer, errOut io.Writer) *cobra.Command {
 	v := &expandCmd{out: out, errOut: errOut}
 
 	cmd := &cobra.Command{
-		Use:     "expand <TENANTNAME> --pool <POOLNAME> --servers <NSERVERS> --volumes <NVOLUMES> --capacity <SIZE> --namespace <TENANTNS>",
+		Use:     "expand <TENANTNAME> --pool <POOLNAME> --servers <NSERVERS> ( --volumes <NVOLUMES> | --volumes-per-server <NVOLUMESPERSERVER> ) --capacity <SIZE> --namespace <TENANTNS>",
 		Short:   "Add capacity to existing tenant",
 		Long:    expandDesc,
 		Example: expandExample,
@@ -73,12 +72,13 @@ func newTenantExpandCmd(out io.Writer, errOut io.Writer) *cobra.Command {
 	f.StringVarP(&v.tenantOpts.PoolName, "pool", "p", "", "name for this pool expansion")
 	f.Int32Var(&v.tenantOpts.Servers, "servers", 0, "total number of pods to add to tenant")
 	f.Int32Var(&v.tenantOpts.Volumes, "volumes", 0, "total number of volumes to add to tenant")
+	f.Int32Var(&v.tenantOpts.VolumesPerServer, "volumes-per-server", 0, "number of volumes in each server in the MinIO tenant")
 	f.StringVar(&v.tenantOpts.Capacity, "capacity", "", "total raw capacity to add to tenant, e.g. 16Ti")
 	f.StringVarP(&v.tenantOpts.StorageClass, "storage-class", "s", helpers.DefaultStorageclass, "storage class for the expanded MinIO tenant pool (can be different than original pool)")
+	f.BoolVar(&v.tenantOpts.DisableAntiAffinity, "enable-host-sharing", false, "[TESTING-ONLY] disable anti-affinity to allow pods to be co-located on a single node (unsupported in production environment)")
 	f.BoolVarP(&v.output, "output", "o", false, "generate MinIO tenant yaml with expansion details")
 
 	cmd.MarkFlagRequired("servers")
-	cmd.MarkFlagRequired("volumes")
 	cmd.MarkFlagRequired("capacity")
 	return cmd
 }
@@ -123,7 +123,13 @@ func (v *expandCmd) run() error {
 		return err
 	}
 	currentCapacity := helpers.TotalCapacity(*t)
-	volumesPerServer := helpers.VolumesPerServer(v.tenantOpts.Volumes, v.tenantOpts.Servers)
+	// Derive Volumes or VolumesPerServer in the absence of the other
+	// Exclusively either variable is guaranteed to exist
+	if v.tenantOpts.Volumes == 0 {
+		v.tenantOpts.Volumes = v.tenantOpts.VolumesPerServer * v.tenantOpts.Servers
+	} else {
+		v.tenantOpts.VolumesPerServer = helpers.VolumesPerServer(v.tenantOpts.Volumes, v.tenantOpts.Servers)
+	}
 	capacityPerVolume, err := helpers.CapacityPerVolume(v.tenantOpts.Capacity, v.tenantOpts.Volumes)
 	if err != nil {
 		return err
@@ -134,7 +140,7 @@ func (v *expandCmd) run() error {
 		v.tenantOpts.PoolName = resources.GeneratePoolName(len(t.Spec.Pools))
 	}
 
-	t.Spec.Pools = append(t.Spec.Pools, resources.Pool(&v.tenantOpts, volumesPerServer, *capacityPerVolume))
+	t.Spec.Pools = append(t.Spec.Pools, resources.Pool(&v.tenantOpts, v.tenantOpts.VolumesPerServer, *capacityPerVolume))
 	expandedCapacity := helpers.TotalCapacity(*t)
 	if !v.output {
 		fmt.Printf(Bold(fmt.Sprintf("\nExpanding Tenant '%s/%s' from %s to %s\n\n", t.ObjectMeta.Name, t.ObjectMeta.Namespace, currentCapacity, expandedCapacity)))

kubectl-minio/cmd/tenant-info.go

@@ -162,13 +162,15 @@ func printTenantInfo(tenant miniov2.Tenant) {
 	}
 
 	t := helpers.GetTable()
-	t.SetHeader([]string{"Pool", "Servers", "Volumes(server)", "Capacity(volume)"})
+	t.SetHeader([]string{"Pool", "Servers", "Volumes per server", "Volumes", "Capacity per volume", "Capacity"})
 	for i, z := range tenant.Spec.Pools {
 		t.Append([]string{
 			strconv.Itoa(i),
 			strconv.Itoa(int(z.Servers)),
 			strconv.Itoa(int(z.VolumesPerServer)),
+			strconv.Itoa(int(z.VolumesPerServer) * int(z.Servers)),
 			humanize.IBytes(uint64(z.VolumeClaimTemplate.Spec.Resources.Requests.Storage().Value())),
+			humanize.IBytes(uint64(z.VolumeClaimTemplate.Spec.Resources.Requests.Storage().Value()) * uint64(z.VolumesPerServer) * uint64(z.Servers)),
 		})
 	}
 	t.Render()

kubectl-minio/go.mod

@@ -38,10 +38,8 @@ require (
 	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/btree v1.1.2 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
@@ -55,17 +53,14 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
-	github.com/lestrrat-go/jwx v1.2.27 // indirect
-	github.com/lestrrat-go/option v1.0.1 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/miekg/dns v1.1.57 // indirect
-	github.com/minio/madmin-go/v3 v3.0.35 // indirect
+	github.com/minio/madmin-go/v2 v2.2.1 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/minio-go/v7 v7.0.65 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
@@ -76,14 +71,10 @@ require (
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
-	github.com/prometheus/prom2json v1.3.3 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/rs/xid v1.5.0 // indirect
-	github.com/safchain/ethtool v0.3.0 // indirect
 	github.com/secure-io/sio-go v0.3.1 // indirect
 	github.com/shirou/gopsutil/v3 v3.23.8 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect