Merge M3 into MCS (#187)

* Merge M3 into MCS

* Fix Linting

* Add missing cluster folder

.golangci.yml

@@ -22,3 +22,7 @@ linters:
 
 service:
   golangci-lint-version: 1.21.0 # use the fixed version to not introduce new linters unexpectedly
+
+run:
+  skip-dirs:
+    - pkg/clientgen

cluster/cluster.go

@@ -0,0 +1,52 @@
+// This file is part of MinIO Kubernetes Cloud
+// Copyright (c) 2019 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cluster
+
+import (
+	operator "github.com/minio/minio-operator/pkg/client/clientset/versioned"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	certutil "k8s.io/client-go/util/cert"
+)
+
+func GetK8sConfig(token string) *rest.Config {
+	// if m3 is running inside k8s by default he will have access to the ca cert from the k8s local authority
+	const (
+		rootCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+	)
+	tlsClientConfig := rest.TLSClientConfig{Insecure: getK8sAPIServerInsecure()}
+	if _, err := certutil.NewPool(rootCAFile); err == nil {
+		tlsClientConfig.CAFile = rootCAFile
+	}
+	config := &rest.Config{
+		Host:            getK8sAPIServer(),
+		TLSClientConfig: tlsClientConfig,
+		APIPath:         "/",
+		BearerToken:     token,
+	}
+	return config
+}
+
+// OperatorClient returns an operator client using GetK8sConfig for its config
+func OperatorClient(token string) (*operator.Clientset, error) {
+	return operator.NewForConfig(GetK8sConfig(token))
+}
+
+// K8sClient returns kubernetes client using GetK8sConfig for its config
+func K8sClient(token string) (*kubernetes.Clientset, error) {
+	return kubernetes.NewForConfig(GetK8sConfig(token))
+}

cluster/config.go

@@ -0,0 +1,168 @@
+// This file is part of MinIO Kubernetes Cloud
+// Copyright (c) 2019 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cluster
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/pkg/env"
+)
+
+var (
+	errCantDetermineMinIOImage = errors.New("can't determine MinIO Image")
+	errCantDetermineMCImage    = errors.New("can't determine MC Image")
+)
+
+func getK8sAPIServer() string {
+	// if m3 is running inside a k8s pod KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT will contain the k8s api server apiServerAddress
+	// if m3 is not running inside k8s by default will look for the k8s api server on localhost:8001 (kubectl proxy)
+	// NOTE: using kubectl proxy is for local development only, since every request send to localhost:8001 will bypass service account authentication
+	// more info here: https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#directly-accessing-the-rest-api
+	// you can override this using M3_K8S_API_SERVER, ie use the k8s cluster from `kubectl config view`
+	host, port := env.Get("KUBERNETES_SERVICE_HOST", ""), env.Get("KUBERNETES_SERVICE_PORT", "")
+	apiServerAddress := "http://localhost:8001"
+	if host != "" && port != "" {
+		apiServerAddress = "https://" + net.JoinHostPort(host, port)
+	}
+	return env.Get(M3K8sAPIServer, apiServerAddress)
+}
+
+// getK8sAPIServerInsecure allow to tell the k8s client to skip TLS certificate verification, ie: when connecting to a k8s cluster
+// that uses certificate not trusted by your machine
+func getK8sAPIServerInsecure() bool {
+	return strings.ToLower(env.Get(m3k8SAPIServerInsecure, "off")) == "on"
+}
+
+// GetNsFromFile assumes mkube is running inside a k8s pod and extract the current namespace from the
+// /var/run/secrets/kubernetes.io/serviceaccount/namespace file
+func GetNsFromFile() string {
+	dat, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+	if err != nil {
+		return "default"
+	}
+	return string(dat)
+}
+
+// This operation will run only once at mkube startup
+var namespace = GetNsFromFile()
+
+// Returns the namespace in which the controller is installed
+func GetNs() string {
+	return env.Get(M3Namespace, namespace)
+}
+
+// getLatestMinIOImage returns the latest docker image for MinIO if found on the internet
+func getLatestMinIOImage(client HTTPClientI) (*string, error) {
+	resp, err := client.Get("https://dl.min.io/server/minio/release/linux-amd64/")
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	var re = regexp.MustCompile(`(?m)\.\/minio\.(RELEASE.*?Z)"`)
+	// look for a single match
+	matches := re.FindAllStringSubmatch(string(body), 1)
+	for i := range matches {
+		release := matches[i][1]
+		dockerImage := fmt.Sprintf("minio/minio:%s", release)
+		return &dockerImage, nil
+	}
+	return nil, errCantDetermineMinIOImage
+}
+
+var latestMinIOImage, errLatestMinIOImage = getLatestMinIOImage(
+	&HTTPClient{
+		Client: &http.Client{
+			Timeout: 4 * time.Second,
+		},
+	})
+
+// GetMinioImage returns the image URL to be used when deploying a MinIO instance, if there is
+// a preferred image to be used (configured via ENVIRONMENT VARIABLES) GetMinioImage will return that
+// if not, GetMinioImage will try to obtain the image URL for the latest version of MinIO and return that
+func GetMinioImage() (*string, error) {
+	image := strings.TrimSpace(env.Get(M3MinioImage, ""))
+	// if there is a preferred image configured by the user we'll always return that
+	if image != "" {
+		return &image, nil
+	}
+	if errLatestMinIOImage != nil {
+		return nil, errLatestMinIOImage
+	}
+	return latestMinIOImage, nil
+}
+
+// GetLatestMinioImage returns the latest image URL on minio repository
+func GetLatestMinioImage(client HTTPClientI) (*string, error) {
+	latestMinIOImage, err := getLatestMinIOImage(client)
+	if err != nil {
+		return nil, err
+	}
+	return latestMinIOImage, nil
+}
+
+// getLatestMCImage returns the latest docker image for MC if found on the internet
+func getLatestMCImage() (*string, error) {
+	// Create an http client with a 4 second timeout
+	client := http.Client{
+		Timeout: 4 * time.Second,
+	}
+	resp, err := client.Get("https://dl.min.io/client/mc/release/linux-amd64/")
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	var re = regexp.MustCompile(`(?m)\.\/mc\.(RELEASE.*?Z)"`)
+	// look for a single match
+	matches := re.FindAllStringSubmatch(string(body), 1)
+	for i := range matches {
+		release := matches[i][1]
+		dockerImage := fmt.Sprintf("minio/mc:%s", release)
+		return &dockerImage, nil
+	}
+	return nil, errCantDetermineMCImage
+}
+
+var latestMCImage, errLatestMCImage = getLatestMCImage()
+
+func GetMCImage() (*string, error) {
+	image := strings.TrimSpace(env.Get(M3MCImage, ""))
+	// if there is a preferred image configured by the user we'll always return that
+	if image != "" {
+		return &image, nil
+	}
+	if errLatestMCImage != nil {
+		return nil, errLatestMCImage
+	}
+	return latestMCImage, nil
+}

cluster/const.go

@@ -0,0 +1,25 @@
+// This file is part of MinIO Kubernetes Cloud
+// Copyright (c) 2019 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cluster
+
+const (
+	M3K8sAPIServer         = "M3_K8S_API_SERVER"
+	m3k8SAPIServerInsecure = "M3_K8S_API_SERVER_INSECURE"
+	M3MinioImage           = "M3_MINIO_IMAGE"
+	M3MCImage              = "M3_MC_IMAGE"
+	M3Namespace            = "M3_NAMESPACE"
+)

cluster/http_client.go

@@ -0,0 +1,40 @@
+// This file is part of MinIO Kubernetes Cloud
+// Copyright (c) 2020 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cluster
+
+import (
+	"net/http"
+)
+
+// HTTPClientI interface with all functions to be implemented
+// by mock when testing, it should include all HttpClient respective api calls
+// that are used within this project.
+type HTTPClientI interface {
+	Get(url string) (resp *http.Response, err error)
+}
+
+// HTTPClient Interface implementation
+//
+// Define the structure of a http client and define the functions that are actually used
+type HTTPClient struct {
+	Client *http.Client
+}
+
+// Get implements http.Client.Get()
+func (c *HTTPClient) Get(url string) (resp *http.Response, err error) {
+	return c.Client.Get(url)
+}

go.mod

@@ -6,25 +6,29 @@ require (
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/elazarl/go-bindata-assetfs v1.0.0
-	github.com/go-openapi/errors v0.19.4
+	github.com/go-openapi/errors v0.19.6
 	github.com/go-openapi/loads v0.19.5
-	github.com/go-openapi/runtime v0.19.12
-	github.com/go-openapi/spec v0.19.7
+	github.com/go-openapi/runtime v0.19.19
+	github.com/go-openapi/spec v0.19.8
 	github.com/go-openapi/strfmt v0.19.5
-	github.com/go-openapi/swag v0.19.8
-	github.com/go-openapi/validate v0.19.7
+	github.com/go-openapi/swag v0.19.9
+	github.com/go-openapi/validate v0.19.10
 	github.com/gorilla/websocket v1.4.2
 	github.com/jessevdk/go-flags v1.4.0
 	github.com/json-iterator/go v1.1.9
 	github.com/minio/cli v1.22.0
 	github.com/minio/mc v0.0.0-20200515235434-3b479cf92ed6
 	github.com/minio/minio v0.0.0-20200603201854-5686a7e27319
 	github.com/minio/minio-go/v6 v6.0.56
+	github.com/minio/minio-operator v0.0.0-20200520220606-60eca6e7beab
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/satori/go.uuid v1.2.0
-	github.com/stretchr/testify v1.5.1
+	github.com/stretchr/testify v1.6.1
 	github.com/unrolled/secure v1.0.7
-	golang.org/x/crypto v0.0.0-20200214034016-1d94cc7ab1c6
-	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e
-	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a
+	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
+	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
+	k8s.io/api v0.18.0
+	k8s.io/apimachinery v0.18.0
+	k8s.io/client-go v0.18.0
 )