diff --git a/cmd/lakectl/cmd/common_helpers.go b/cmd/lakectl/cmd/common_helpers.go index 20aeb4f11f7..f9a7675a78c 100644 --- a/cmd/lakectl/cmd/common_helpers.go +++ b/cmd/lakectl/cmd/common_helpers.go @@ -2,12 +2,14 @@ package cmd import ( "bytes" + "encoding/base64" "encoding/json" "errors" "fmt" "io" "net/http" "os" + "regexp" "strings" "text/template" "time" @@ -27,6 +29,8 @@ var noColorRequested = false // ErrInvalidValueInList is an error returned when a parameter of type list contains an empty string var ErrInvalidValueInList = errors.New("empty string in list") +var accessKeyRegexp = regexp.MustCompile(`^AKIA[I|J][A-Z0-9]{14}Q$`) + const ( LakectlInteractive = "LAKECTL_INTERACTIVE" LakectlInteractiveDisable = "no" @@ -267,3 +271,16 @@ func MustParsePathURI(name, s string) *uri.URI { } return u } + +func IsValidAccessKeyID(accessKeyID string) bool { + return accessKeyRegexp.MatchString(accessKeyID) +} + +func IsValidSecretAccessKey(secretAccessKey string) bool { + return IsBase64(secretAccessKey) && len(secretAccessKey) == 40 +} + +func IsBase64(s string) bool { + _, err := base64.StdEncoding.DecodeString(s) + return err == nil +} diff --git a/cmd/lakectl/cmd/common_helpers_test.go b/cmd/lakectl/cmd/common_helpers_test.go new file mode 100644 index 00000000000..88fcc155741 --- /dev/null +++ b/cmd/lakectl/cmd/common_helpers_test.go @@ -0,0 +1,52 @@ +package cmd + +import "testing" + +func TestIsValidAccessKeyID(t *testing.T) { + type args struct { + accessKeyID string + } + tests := []struct { + name string + args args + want bool + }{ + {name: "valid access key id", args: args{accessKeyID: "AKIAJ12ZZZZZZZZZZZZQ"}, want: true}, + {name: "access key id with lower case char", args: args{accessKeyID: "AKIAJ12zZZZZZZZZZZZQ"}, want: false}, + {name: "access key id with invalid char", args: args{accessKeyID: "AKIAJ12!ZZZZZZZZZZZQ"}, want: false}, + {name: "access key id with extra char", args: args{accessKeyID: "AKIAJ123ZZZZZZZZZZZZQ"}, want: false}, + {name: "access key id with missing char", args: args{accessKeyID: "AKIAJ1ZZZZZZZZZZZZQ"}, want: false}, + {name: "access key id with wrong prefix", args: args{accessKeyID: "AKIAM12ZZZZZZZZZZZZQ"}, want: false}, + {name: "access key id with wrong suffiix", args: args{accessKeyID: "AKIAJ12ZZZZZZZZZZZZA"}, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsValidAccessKeyID(tt.args.accessKeyID); got != tt.want { + t.Errorf("IsValidAccessKeyID() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsValidSecretAccessKey(t *testing.T) { + type args struct { + secretAccessKey string + } + tests := []struct { + name string + args args + want bool + }{ + {name: "valid secret access key", args: args{secretAccessKey: "TQG5JcovOozCGJnIRmIKH7Flq1tLxrUbyi9/WmJy"}, want: true}, + {name: "secret access key id with invalid char", args: args{secretAccessKey: "!QG5JcovOozCGJnIRmIKH7Flq1tLxrUbyi9/WmJy"}, want: false}, + {name: "secret access key id with extra char", args: args{secretAccessKey: "aTQG5JcovOozCGJnIRmIKH7Flq1tLxrUbyi9/WmJy"}, want: false}, + {name: "secret access key id with missing char", args: args{secretAccessKey: "QG5JcovOozCGJnIRmIKH7Flq1tLxrUbyi9/WmJy"}, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsValidSecretAccessKey(tt.args.secretAccessKey); got != tt.want { + t.Errorf("IsValidSecretAccessKey() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/lakectl/cmd/doctor.go b/cmd/lakectl/cmd/doctor.go new file mode 100644 index 00000000000..9582384caab --- /dev/null +++ b/cmd/lakectl/cmd/doctor.go @@ -0,0 +1,141 @@ +package cmd + +import ( + "context" + "errors" + "net/url" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/treeverse/lakefs/pkg/api" +) + +type Detailed interface { + GetDetails() string +} + +type CredentialsError struct { + Message string + Details string +} + +func (e *CredentialsError) Error() string { return e.Message } + +func (e *CredentialsError) GetDetails() string { + return e.Message + "\n" + e.Details +} + +type WrongEndpointURIError struct { + Message string + Details string +} + +func (e *WrongEndpointURIError) Error() string { return e.Message } + +func (e *WrongEndpointURIError) GetDetails() string { + return e.Message + "\n" + e.Details +} + +type UnknownConfigError struct { + Message string + Details string +} + +func (e *UnknownConfigError) Error() string { return e.Message } + +func (e *UnknownConfigError) GetDetails() string { + return e.Message + "\n" + e.Details +} + +type UserMessage struct { + Message string +} + +var detailedErrorTemplate = `{{ .Message |red }} +{{ .Details }} +` +var errorTemplate = `{{ .Message |red }} +` + +var successMessageTemplate = `{{ .Message | green}} +` + +var analayzingMessageTemplate = `{{ .Message }} +` + +// doctorCmd represents the doctor command +var doctorCmd = &cobra.Command{ + Use: "doctor", + Short: "Run a basic diagnosis of the LakeFS configuration", + Run: func(cmd *cobra.Command, args []string) { + err := ListRepositoriesAndAnalyze(cmd.Context()) + if err == nil { + Write(successMessageTemplate, &UserMessage{"Valid configuration"}) + return + } + + if detailedErr, ok := err.(Detailed); ok { + Write(detailedErrorTemplate, detailedErr) + } else { + Write(errorTemplate, err) + } + + accessKeyID := cfg.Values.Credentials.AccessKeyID + if !IsValidAccessKeyID(accessKeyID) { + Write(analayzingMessageTemplate, &UserMessage{"access_key_id value looks suspicious: " + accessKeyID}) + } + + secretAccessKey := cfg.Values.Credentials.SecretAccessKey + if !IsValidSecretAccessKey(secretAccessKey) { + Write(analayzingMessageTemplate, &UserMessage{"secret_access_key value looks suspicious..."}) + } + + serverEndpoint := cfg.Values.Server.EndpointURL + if !strings.HasSuffix(serverEndpoint, api.BaseURL) { + Write(analayzingMessageTemplate, &UserMessage{"Suspicious URI format for server.endpoint_url: " + serverEndpoint + " doesn't end with: `" + api.BaseURL + "`."}) + } + }, +} + +func ListRepositoriesAndAnalyze(ctx context.Context) error { + configFileName := viper.GetViper().ConfigFileUsed() + msgOnErrUnknownConfig := "It looks like you have a problem with your `" + configFileName + "` file." + msgOnErrWrongEndpointURI := "It looks like endpoint url is wrong." + msgOnErrCredential := "It seems like the `access_key_id` or `secret_access_key` you supplied are wrong." + + // getClient might die on url.Parse error, so check it first. + serverEndpoint := cfg.Values.Server.EndpointURL + _, err := url.Parse(serverEndpoint) + if err != nil { + return &WrongEndpointURIError{msgOnErrWrongEndpointURI, err.Error()} + } + client := getClient() + resp, err := client.ListRepositoriesWithResponse(ctx, &api.ListRepositoriesParams{}) + + switch { + case err != nil: + urlErr := &url.Error{} + if errors.As(err, &urlErr) { + return &WrongEndpointURIError{msgOnErrWrongEndpointURI, err.Error()} + } + return &UnknownConfigError{msgOnErrUnknownConfig, err.Error()} + case resp == nil: + break + case resp.JSON200 != nil: + return nil + case resp.JSON401 != nil: + return &CredentialsError{msgOnErrCredential, resp.JSON401.Message} + // In case we get the "not found" HTML page (the status is 200 and not 404 in this case). + case resp.HTTPResponse != nil && resp.HTTPResponse.StatusCode == 200: + return &WrongEndpointURIError{msgOnErrWrongEndpointURI, ""} + case resp.JSONDefault != nil: + return &UnknownConfigError{msgOnErrUnknownConfig, resp.JSONDefault.Message} + } + return &UnknownConfigError{msgOnErrUnknownConfig, "An unknown error accourd while trying to analyzing LakeCtl configuration."} +} + +//nolint:gochecknoinits +func init() { + rootCmd.AddCommand(doctorCmd) +} diff --git a/docs/reference/commands.md b/docs/reference/commands.md index de5441906fd..ff7bc15aa70 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -1735,6 +1735,23 @@ lakectl docs [outfile] [flags] +### lakectl doctor + +Run a basic diagnosis of the LakeFS configuration + +``` +lakectl doctor [flags] +``` + +#### Options +{:.no_toc} + +``` + -h, --help help for doctor +``` + + + ### lakectl fs View and manipulate objects diff --git a/nessie/doctor_test.go b/nessie/doctor_test.go new file mode 100644 index 00000000000..40d595f3871 --- /dev/null +++ b/nessie/doctor_test.go @@ -0,0 +1,25 @@ +package nessie + +import ( + "os" + "testing" + + "github.com/spf13/viper" +) + +func TestDoctor(t *testing.T) { + accessKeyID := viper.GetString("access_key_id") + secretAccessKey := viper.GetString("secret_access_key") + endPointURL := viper.GetString("endpoint_url") + "/api/v1" + defaultConfigPath := "/root/.lakectl.yaml" + os.Create(defaultConfigPath) + RunCmdAndVerifySuccessWithFile(t, LakectlWithParams(accessKeyID, secretAccessKey, endPointURL)+" doctor", false, "lakectl_doctor_ok", emptyVars) + RunCmdAndVerifyFailureWithFile(t, lakectlLocation()+" doctor -c not_exits.yaml", false, "lakectl_doctor_not_exists_file", emptyVars) + RunCmdAndVerifySuccessWithFile(t, LakectlWithParams(accessKeyID, secretAccessKey, endPointURL+"1")+" doctor", false, "lakectl_doctor_wrong_endpoint", emptyVars) + RunCmdAndVerifySuccessWithFile(t, LakectlWithParams(accessKeyID, secretAccessKey, "wrong_uri")+" doctor", false, "lakectl_doctor_wrong_uri_format_endpoint", emptyVars) + RunCmdAndVerifySuccessWithFile(t, LakectlWithParams("AKIAJZZZZZZZZZZZZZZQ", secretAccessKey, endPointURL)+" doctor", false, "lakectl_doctor_with_wrong_credentials", emptyVars) + RunCmdAndVerifySuccessWithFile(t, LakectlWithParams("AKIAJOI!COZ5JBYHCSDQ", secretAccessKey, endPointURL)+" doctor", false, "lakectl_doctor_with_suspicious_access_key_id", emptyVars) + RunCmdAndVerifySuccessWithFile(t, LakectlWithParams(accessKeyID, "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", endPointURL)+" doctor", false, "lakectl_doctor_with_wrong_credentials", emptyVars) + RunCmdAndVerifySuccessWithFile(t, LakectlWithParams(accessKeyID, "TQG5JcovOozCGJnIRmIKH7Flq1tLxnuByi9/WmJ!", endPointURL)+" doctor", false, "lakectl_doctor_with_suspicious_secret_access_key", emptyVars) + +} diff --git a/nessie/golden/lakectl_doctor_not_exists_file.golden b/nessie/golden/lakectl_doctor_not_exists_file.golden new file mode 100644 index 00000000000..a164e3f11ad --- /dev/null +++ b/nessie/golden/lakectl_doctor_not_exists_file.golden @@ -0,0 +1,2 @@ +error reading configuration file: open not_exits.yaml: no such file or directory +Error executing command. diff --git a/nessie/golden/lakectl_doctor_ok.golden b/nessie/golden/lakectl_doctor_ok.golden new file mode 100644 index 00000000000..759761d0da5 --- /dev/null +++ b/nessie/golden/lakectl_doctor_ok.golden @@ -0,0 +1 @@ +Valid configuration diff --git a/nessie/golden/lakectl_doctor_with_suspicious_access_key_id.golden b/nessie/golden/lakectl_doctor_with_suspicious_access_key_id.golden new file mode 100644 index 00000000000..dc72433a405 --- /dev/null +++ b/nessie/golden/lakectl_doctor_with_suspicious_access_key_id.golden @@ -0,0 +1,3 @@ +It seems like the `access_key_id` or `secret_access_key` you supplied are wrong. +error authenticating request +access_key_id value looks suspicious: AKIAJOI!COZ5JBYHCSDQ diff --git a/nessie/golden/lakectl_doctor_with_suspicious_secret_access_key.golden b/nessie/golden/lakectl_doctor_with_suspicious_secret_access_key.golden new file mode 100644 index 00000000000..edc944883aa --- /dev/null +++ b/nessie/golden/lakectl_doctor_with_suspicious_secret_access_key.golden @@ -0,0 +1,3 @@ +It seems like the `access_key_id` or `secret_access_key` you supplied are wrong. +error authenticating request +secret_access_key value looks suspicious... diff --git a/nessie/golden/lakectl_doctor_with_wrong_credentials.golden b/nessie/golden/lakectl_doctor_with_wrong_credentials.golden new file mode 100644 index 00000000000..1a092d13065 --- /dev/null +++ b/nessie/golden/lakectl_doctor_with_wrong_credentials.golden @@ -0,0 +1,2 @@ +It seems like the `access_key_id` or `secret_access_key` you supplied are wrong. +error authenticating request diff --git a/nessie/golden/lakectl_doctor_wrong_endpoint.golden b/nessie/golden/lakectl_doctor_wrong_endpoint.golden new file mode 100644 index 00000000000..cd7db79acc6 --- /dev/null +++ b/nessie/golden/lakectl_doctor_wrong_endpoint.golden @@ -0,0 +1,3 @@ +It looks like endpoint url is wrong. + +Suspicious URI format for server.endpoint_url: http://lakefs:8000/api/v11 doesn't end with: `/api/v1`. diff --git a/nessie/golden/lakectl_doctor_wrong_uri_format_endpoint.golden b/nessie/golden/lakectl_doctor_wrong_uri_format_endpoint.golden new file mode 100644 index 00000000000..622536ff57a --- /dev/null +++ b/nessie/golden/lakectl_doctor_wrong_uri_format_endpoint.golden @@ -0,0 +1,3 @@ +It looks like endpoint url is wrong. +Get "/wrong_uri/repositories": unsupported protocol scheme "" +Suspicious URI format for server.endpoint_url: wrong_uri doesn't end with: `/api/v1`. diff --git a/nessie/golden/lakectl_help.golden b/nessie/golden/lakectl_help.golden index 1221e5d3439..c671531491d 100644 --- a/nessie/golden/lakectl_help.golden +++ b/nessie/golden/lakectl_help.golden @@ -16,6 +16,7 @@ Available Commands: config Create/update local lakeFS configuration dbt Integration with dbt commands diff Show changes between two commits, or the currently uncommitted changes + doctor Run a basic diagnosis of the LakeFS configuration fs View and manipulate objects gc Manage garbage collection configuration help Help about any command diff --git a/nessie/lakectl_util.go b/nessie/lakectl_util.go index 168123863dc..b2a58d28181 100644 --- a/nessie/lakectl_util.go +++ b/nessie/lakectl_util.go @@ -22,16 +22,20 @@ func lakectlLocation() string { return viper.GetString("lakectl_dir") + "/lakectl" } -func Lakectl() string { +func LakectlWithParams(accessKeyID, secretAccessKey, endPointURL string) string { lakectlCmdline := - "LAKECTL_CREDENTIALS_ACCESS_KEY_ID=" + viper.GetString("access_key_id") + - " LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY=" + viper.GetString("secret_access_key") + - " LAKECTL_SERVER_ENDPOINT_URL=" + viper.GetString("endpoint_url") + + "LAKECTL_CREDENTIALS_ACCESS_KEY_ID=" + accessKeyID + + " LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY=" + secretAccessKey + + " LAKECTL_SERVER_ENDPOINT_URL=" + endPointURL + " " + lakectlLocation() return lakectlCmdline } +func Lakectl() string { + return LakectlWithParams(viper.GetString("access_key_id"), viper.GetString("secret_access_key"), viper.GetString("endpoint_url")) +} + func runShellCommand(command string, isTerminal bool) ([]byte, error) { fmt.Println("Testing '", command, "'") var cmd *exec.Cmd