Large MC Update (new encryption flags, functional test suite, removal…

… of session code, minor cleanup, vuln. updates ) (#4882)

.github/workflows/go.yml

@@ -49,6 +49,9 @@ jobs:
           ENABLE_HTTPS: 1
           MINIO_CI_CD: 1
           SERVER_ENDPOINT: localhost:9000
+          MC_TEST_ENABLE_HTTPS: true
+          MC_TEST_SKIP_INSECURE: true
+          MC_TEST_SKIP_BUILD: true
         run: |
           wget https://dl.min.io/server/minio/release/linux-amd64/minio && chmod +x minio
           mkdir -p ~/.minio/certs/ && cp testdata/localhost.crt ~/.minio/certs/public.crt && cp testdata/localhost.key ~/.minio/certs/private.key

Makefile

@@ -43,7 +43,7 @@ test: verifiers build
 	@echo "Running unit tests"
 	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 	@echo "Running functional tests"
-	@(env bash $(PWD)/functional-tests.sh)
+	@GO111MODULE=on MC_TEST_RUN_FULL_SUITE=true go test -race -v --timeout 20m ./... -run Test_FullSuite
 
 test-race: verifiers build
 	@echo "Running unit tests under -race"
@@ -54,7 +54,7 @@ verify:
 	@echo "Verifying build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/mc 1>/dev/null
 	@echo "Running functional tests"
-	@(env bash $(PWD)/functional-tests.sh)
+	@GO111MODULE=on MC_TEST_RUN_FULL_SUITE=true go test -race -v --timeout 20m ./... -run Test_FullSuite
 
 # Builds mc locally.
 build: checks

README.md

@@ -1,41 +1,53 @@
 # MinIO Client Quickstart Guide
 [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/mc)](https://goreportcard.com/report/minio/mc) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/mc.svg?maxAge=604800)](https://hub.docker.com/r/minio/mc/) [![license](https://img.shields.io/badge/license-AGPL%20V3-blue)](https://github.com/minio/mc/blob/master/LICENSE)
 
+# Documentation
+- [MC documentation](https://min.io/docs/minio/linux/reference/minio-mc.html)
+
 MinIO Client (mc) provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff, find etc. It supports filesystems and Amazon S3 compatible cloud storage service (AWS Signature v2 and v4).
 
 ```
-alias       set, remove and list aliases in configuration file
-ls          list buckets and objects
-mb          make a bucket
-rb          remove a bucket
-cp          copy objects
-mirror      synchronize object(s) to a remote site
-cat         display object contents
-head        display first 'n' lines of an object
-pipe        stream STDIN to an object
-share       generate URL for temporary access to an object
-find        search for objects
-sql         run sql queries on objects
-stat        show object metadata
-mv          move objects
-tree        list buckets and objects in a tree format
-du          summarize disk usage recursively
-retention   set retention for object(s)
-legalhold   set legal hold for object(s)
-diff        list differences in object name, size, and date between two buckets
-rm          remove objects
-encrypt     manage bucket encryption config
-event       manage object notifications
-watch       listen for object notification events
-undo        undo PUT/DELETE operations
-anonymous   manage anonymous access to buckets and objects
-tag         manage tags for bucket(s) and object(s)
-ilm         manage bucket lifecycle
-version     manage bucket versioning
-replicate   configure server side bucket replication
-admin       manage MinIO servers
-update      update mc to latest release
-ping        perform liveness check
+  alias      manage server credentials in configuration file
+  admin      manage MinIO servers
+  anonymous  manage anonymous access to buckets and objects
+  batch      manage batch jobs
+  cp         copy objects
+  cat        display object contents
+  diff       list differences in object name, size, and date between two buckets
+  du         summarize disk usage recursively
+  encrypt    manage bucket encryption config
+  event      manage object notifications
+  find       search for objects
+  get        get s3 object to local
+  head       display first 'n' lines of an object
+  ilm        manage bucket lifecycle
+  idp        manage MinIO IDentity Provider server configuration
+  license    license related commands
+  legalhold  manage legal hold for object(s)
+  ls         list buckets and objects
+  mb         make a bucket
+  mv         move objects
+  mirror     synchronize object(s) to a remote site
+  od         measure single stream upload and download
+  ping       perform liveness check
+  pipe       stream STDIN to an object
+  put        upload an object to a bucket
+  quota      manage bucket quota
+  rm         remove object(s)
+  retention  set retention for object(s)
+  rb         remove a bucket
+  replicate  configure server side bucket replication
+  ready      checks if the cluster is ready or not
+  sql        run sql queries on objects
+  stat       show object metadata
+  support    support related commands
+  share      generate URL for temporary access to an object
+  tree       list buckets and objects in a tree format
+  tag        manage tags for bucket and object(s)
+  undo       undo PUT/DELETE operations
+  update     update mc to latest release
+  version    manage bucket versioning
+  watch      listen for object notification events
 ```
 
 ## Docker Container
@@ -176,9 +188,6 @@ Get your AccessKeyID and SecretAccessKey by following [Google Credentials Guide]
 mc alias set gcs  https://storage.googleapis.com BKIKJAA5BMMU2RHO6IBB V8f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12
 ```
 
-### Example - IBM Cloud Object Storage
-See [the complete guide](docs/minio-client-complete-guide.md) for IBM instructions.
-
 ## Test Your Setup
 `mc` is pre-configured with https://play.min.io, aliased as "play". It is a hosted MinIO server for testing and development purpose.  To test Amazon S3, simply replace "play" with "s3" or the alias you used at the time of setup.
 
@@ -236,11 +245,6 @@ admin    config   diff     find     ls       mirror   policy   session  sql
 cat      cp       event    head     mb       pipe     rm       share    stat     version
 ```
 
-## Explore Further
-- [MinIO Client Complete Guide](https://min.io/docs/minio/linux/reference/minio-mc.html?ref=gh)
-- [MinIO Quickstart Guide](https://min.io/docs/minio/linux/index.html#quickstart-for-linux?ref=gh)
-- [The MinIO documentation website](https://min.io/docs/minio/linux/index.html?ref=gh)
-
 ## Contribute to MinIO Project
 Please follow MinIO [Contributor's Guide](https://github.com/minio/mc/blob/master/CONTRIBUTING.md)
 

cmd/admin-kms-key-create.go

@@ -30,7 +30,7 @@ import (
 
 var adminKMSCreateKeyCmd = cli.Command{
 	Name:         "create",
-	Usage:        "creates a new master key at the KMS",
+	Usage:        "creates a new master KMS key",
 	Action:       mainAdminKMSCreateKey,
 	OnUsageError: onUsageError,
 	Before:       setGlobalsFromContext,

cmd/cat-main.go

@@ -64,7 +64,7 @@ var catCmd = cli.Command{
 	Action:       mainCat,
 	OnUsageError: onUsageError,
 	Before:       setGlobalsFromContext,
-	Flags:        append(append(catFlags, ioFlags...), globalFlags...),
+	Flags:        append(append(catFlags, encCFlag), globalFlags...),
 	CustomHelpTemplate: `NAME:
   {{.HelpName}} - {{.Usage}}
 
@@ -74,8 +74,6 @@ USAGE:
 FLAGS:
   {{range .VisibleFlags}}{{.}}
   {{end}}
-ENVIRONMENT VARIABLES:
-  MC_ENCRYPT_KEY:  list of comma delimited prefix=secret values
 
 EXAMPLES:
   1. Stream an object from Amazon S3 cloud storage to mplayer standard input.
@@ -88,11 +86,11 @@ EXAMPLES:
      {{.Prompt}} {{.HelpName}} part.* > complete.img
 
   4. Save an encrypted object from Amazon S3 cloud storage to a local file.
-     {{.Prompt}} {{.HelpName}} --encrypt-key 's3/mysql-backups=32byteslongsecretkeymustbegiven1' s3/mysql-backups/backups-201810.gz > /mnt/data/recent.gz
+     {{.Prompt}} {{.HelpName}} --enc-c "play/my-bucket/=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA" s3/mysql-backups/backups-201810.gz > /mnt/data/recent.gz
 
   5. Display the content of encrypted object. In case the encryption key contains non-printable character like tab, pass the
      base64 encoded string as key.
-     {{.Prompt}} {{.HelpName}} --encrypt-key "play/my-bucket/=MzJieXRlc2xvbmdzZWNyZXRrZQltdXN0YmVnaXZlbjE="  play/my-bucket/my-object
+     {{.Prompt}} {{.HelpName}} --enc-c "play/my-bucket/=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA" play/my-bucket/my-object
 
   6. Display the content of an object 10 days earlier
      {{.Prompt}} {{.HelpName}} --rewind 10d play/my-bucket/my-object
@@ -323,15 +321,12 @@ func mainCat(cliCtx *cli.Context) error {
 	ctx, cancelCat := context.WithCancel(globalContext)
 	defer cancelCat()
 
-	// Parse encryption keys per command.
-	encKeyDB, err := getEncKeys(cliCtx)
+	encKeyDB, err := validateAndCreateEncryptionKeys(cliCtx)
 	fatalIf(err, "Unable to parse encryption keys.")
 
 	// check 'cat' cli arguments.
 	o := parseCatSyntax(cliCtx)
 
-	// Set command flags from context.
-
 	// handle std input data.
 	if o.stdinMode {
 		fatalIf(catOut(os.Stdin, -1).Trace(), "Unable to read from standard input.")

cmd/client-fs_linux.go

@@ -68,8 +68,7 @@ func IsDeleteEvent(event notify.Event) bool {
 	return false
 }
 
-// getXAttr fetches the extended attribute for a particular key on
-// file
+// getXAttr fetches the extended attribute for a particular key on file
 func getXAttr(path, key string) (string, error) {
 	data, e := xattr.Get(path, key)
 	if e != nil {
@@ -81,8 +80,7 @@ func getXAttr(path, key string) (string, error) {
 	return hex.EncodeToString(data), nil
 }
 
-// getAllXattrs returns the extended attributes for a file if supported
-// by the OS
+// getAllXattrs returns the extended attributes for a file if supported by the OS
 func getAllXattrs(path string) (map[string]string, error) {
 	xMetadata := make(map[string]string)
 	list, e := xattr.List(path)

cmd/client-s3.go

@@ -92,6 +92,8 @@ const (
 	AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date"
 	// AmzObjectLockLegalHold sets object lock legal hold
 	AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold"
+	amzObjectSSEKMSKeyID   = "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"
+	amzObjectSSE           = "X-Amz-Server-Side-Encryption"
 )
 
 type dialContext func(ctx context.Context, network, addr string) (net.Conn, error)

cmd/common-methods.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"context"
-	"encoding/base64"
 	"errors"
 	"io"
 	"net/http"
@@ -33,73 +32,12 @@ import (
 	"golang.org/x/net/http/httpguts"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/cli"
 	"github.com/minio/mc/pkg/probe"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/pkg/v2/env"
 )
 
-// decode if the key is encoded key and returns the key
-func getDecodedKey(sseKeys string) (key string, err *probe.Error) {
-	keyString := ""
-	for i, sse := range strings.Split(sseKeys, ",") {
-		if i > 0 {
-			keyString = keyString + ","
-		}
-		sseString, err := parseKey(sse)
-		if err != nil {
-			return "", err
-		}
-		keyString = keyString + sseString
-	}
-	return keyString, nil
-}
-
-// Validate the key
-func parseKey(sseKeys string) (sse string, err *probe.Error) {
-	encryptString := strings.SplitN(sseKeys, "=", 2)
-	if len(encryptString) < 2 {
-		return "", probe.NewError(errors.New("SSE-C prefix should be of the form prefix1=key1,... "))
-	}
-
-	secretValue := encryptString[1]
-	if len(secretValue) == 32 {
-		return sseKeys, nil
-	}
-	decodedString, e := base64.StdEncoding.DecodeString(secretValue)
-	if e != nil || len(decodedString) != 32 {
-		return "", probe.NewError(errors.New("Encryption key should be 32 bytes plain text key or 44 bytes base64 encoded key"))
-	}
-	return encryptString[0] + "=" + string(decodedString), nil
-}
-
-// parse and return encryption key pairs per alias.
-func getEncKeys(ctx *cli.Context) (map[string][]prefixSSEPair, *probe.Error) {
-	sseServer := ctx.String("encrypt")
-	var sseKeys string
-	if keyPrefix := ctx.String("encrypt-key"); keyPrefix != "" {
-		if sseServer != "" && strings.Contains(keyPrefix, sseServer) {
-			return nil, errConflictSSE(sseServer, keyPrefix).Trace(ctx.Args()...)
-		}
-		sseKeys = keyPrefix
-	}
-	var err *probe.Error
-	if sseKeys != "" {
-		sseKeys, err = getDecodedKey(sseKeys)
-		if err != nil {
-			return nil, err.Trace(sseKeys)
-		}
-	}
-
-	encKeyDB, err := parseAndValidateEncryptionKeys(sseKeys, sseServer)
-	if err != nil {
-		return nil, err.Trace(sseKeys)
-	}
-
-	return encKeyDB, nil
-}
-
 // Check if the passed URL represents a folder. It may or may not exist yet.
 // If it exists, we can easily check if it is a folder, if it doesn't exist,
 // we can guess if the url is a folder from how it looks.