Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

allow lakectl local to be "git data" #7618

Merged
merged 9 commits into from
Apr 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 13 additions & 16 deletions cmd/lakectl/cmd/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ func localDiff(ctx context.Context, client apigen.ClientWithResponsesInterface,
}

func localHandleSyncInterrupt(ctx context.Context, idx *local.Index, operation string) context.Context {
cmdName := ctx.Value(lakectlLocalCommandNameKey).(string)
ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
go func() {
defer stop()
Expand All @@ -87,27 +88,28 @@ func localHandleSyncInterrupt(ctx context.Context, idx *local.Index, operation s
if err != nil {
WriteTo("{{.Error|red}}\n", struct{ Error string }{Error: "Failed to write failed operation to index file."}, os.Stderr)
}
Die(`Operation was canceled, local data may be incomplete.
Use "lakectl local checkout..." to sync with the remote.`, 1)
DieFmt(`Operation was canceled, local data may be incomplete.
Use "%s checkout..." to sync with the remote.`, cmdName)
}()
return ctx
}

func dieOnInterruptedOperation(interruptedOperation LocalOperation, force bool) {
func dieOnInterruptedOperation(ctx context.Context, interruptedOperation LocalOperation, force bool) {
cmdName := ctx.Value(lakectlLocalCommandNameKey).(string)
if !force && interruptedOperation != "" {
switch interruptedOperation {
case commitOperation:
Die(`Latest commit operation was interrupted, data may be incomplete.
Use "lakectl local commit..." to commit your latest changes or "lakectl local pull... --force" to sync with the remote.`, 1)
DieFmt(`Latest commit operation was interrupted, data may be incomplete.
Use "%s commit..." to commit your latest changes or "lakectl local pull... --force" to sync with the remote.`, cmdName)
case checkoutOperation:
Die(`Latest checkout operation was interrupted, local data may be incomplete.
Use "lakectl local checkout..." to sync with the remote.`, 1)
DieFmt(`Latest checkout operation was interrupted, local data may be incomplete.
Use "%s checkout..." to sync with the remote.`, cmdName)
case pullOperation:
Die(`Latest pull operation was interrupted, local data may be incomplete.
Use "lakectl local pull... --force" to sync with the remote.`, 1)
DieFmt(`Latest pull operation was interrupted, local data may be incomplete.
Use "%s pull... --force" to sync with the remote.`, cmdName)
case cloneOperation:
Die(`Latest clone operation was interrupted, local data may be incomplete.
Use "lakectl local checkout..." to sync with the remote or run "lakectl local clone..." with a different directory to sync with the remote.`, 1)
DieFmt(`Latest clone operation was interrupted, local data may be incomplete.
Use "%s checkout..." to sync with the remote or run "lakectl local clone..." with a different directory to sync with the remote.`, cmdName)
default:
panic(fmt.Errorf("found an unknown interrupted operation in the index file: %s- %w", interruptedOperation, ErrUnknownOperation))
}
Expand All @@ -118,8 +120,3 @@ var localCmd = &cobra.Command{
Use: "local",
Short: "Sync local directories with lakeFS paths",
}

//nolint:gochecknoinits
func init() {
rootCmd.AddCommand(localCmd)
}
60 changes: 60 additions & 0 deletions cmd/lakectl/cmd/local_install.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package cmd

import (
"os"
"path"
"path/filepath"
"runtime"

"github.com/mitchellh/go-homedir"

"github.com/spf13/cobra"
)

func currentExecutable() string {
ex, err := os.Executable()
if err != nil {
DieErr(err)
}
absolute, err := filepath.Abs(ex)
if err != nil {
DieErr(err)
}
return absolute
}

var installGitPluginCmd = &cobra.Command{
Use: "install-git-plugin <directory>",
Short: "set up `git data` (directory must exist and be in $PATH)",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add a long description here? I think in this case it's important to describe exactly how to use it

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added!

Long: "Add a symlink to lakectl named `git-data`.\n" +
"This allows calling `git data` and having it act as the `lakectl local` command\n" +
"(as long as the symlink is within the executing users' $PATH environment variable",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
installDir, err := homedir.Expand(args[0])
if err != nil {
DieFmt("could not get directory path %s: %s\n", args[0], err.Error())
}
info, err := os.Stat(installDir)
if err != nil {
DieFmt("could not check directory %s: %s\n", installDir, err.Error())
}
if !info.IsDir() {
DieFmt("%s: not a directory.\n", installDir)
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we also want to verify it's in the path?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's possibly error prone: git is sometimes a script that sets up environment variables before calling the executable, so we can't guarantee the current $PATH is the one used when executing.

fullPath := path.Join(installDir, "git-data")
if runtime.GOOS == "windows" {
fullPath += ".exe"
}
err = os.Symlink(currentExecutable(), fullPath)
if err != nil {
DieFmt("could not create link %s: %s\n", fullPath, err.Error())
}
},
}

//nolint:gochecknoinits
func init() {
rootCmd.AddCommand(installGitPluginCmd)
}
2 changes: 1 addition & 1 deletion cmd/lakectl/cmd/local_pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ var localPullCmd = &cobra.Command{
DieErr(err)
}

dieOnInterruptedOperation(LocalOperation(idx.ActiveOperation), force)
dieOnInterruptedOperation(cmd.Context(), LocalOperation(idx.ActiveOperation), force)

currentBase := remote.WithRef(idx.AtHead)
// make sure no local changes
Expand Down
2 changes: 1 addition & 1 deletion cmd/lakectl/cmd/local_status.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ var localStatusCmd = &cobra.Command{
DieErr(err)
}

dieOnInterruptedOperation(LocalOperation(idx.ActiveOperation), false)
dieOnInterruptedOperation(cmd.Context(), LocalOperation(idx.ActiveOperation), false)

remoteBase := remote.WithRef(idx.AtHead)
client := getClient()
Expand Down
153 changes: 87 additions & 66 deletions cmd/lakectl/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import (
"golang.org/x/exp/slices"
)

type lakectlLocalContextKey string

const (
DefaultMaxIdleConnsPerHost = 100
// version templates
Expand All @@ -49,6 +51,7 @@ lakeFS version: {{.LakeFSVersion}}
Get the latest release {{ .UpgradeURL|blue }}
{{- end }}
`
lakectlLocalCommandNameKey lakectlLocalContextKey = "lakectl-local-command-name"
)

// Configuration is the user-visible configuration structure in Golang form.
Expand Down Expand Up @@ -282,63 +285,63 @@ func getKV(cmd *cobra.Command, name string) (map[string]string, error) { //nolin
return kv, nil
}

// rootCmd represents the base command when called without any sub-commands
var rootCmd = &cobra.Command{
Use: "lakectl",
Short: "A cli tool to explore manage and work with lakeFS",
Long: `lakectl is a CLI tool allowing exploration and manipulation of a lakeFS environment`,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
logging.SetLevel(logLevel)
logging.SetOutputFormat(logFormat)
err := logging.SetOutputs(logOutputs, 0, 0)
if err != nil {
DieFmt("Failed to setup logging: %s", err)
}
if noColorRequested {
DisableColors()
}
if cmd == configCmd {
return
}
func rootPreRun(cmd *cobra.Command, _ []string) {
logging.SetLevel(logLevel)
logging.SetOutputFormat(logFormat)
err := logging.SetOutputs(logOutputs, 0, 0)
if err != nil {
DieFmt("Failed to setup logging: %s", err)
}
if noColorRequested {
DisableColors()
}
if cmd == configCmd {
return
}

if cfgErr == nil {
logging.ContextUnavailable().
WithField("file", viper.ConfigFileUsed()).
Debug("loaded configuration from file")
} else if errors.As(cfgErr, &viper.ConfigFileNotFoundError{}) {
if cfgFile != "" {
// specific message in case the file isn't found
DieFmt("config file not found, please run \"lakectl config\" to create one\n%s\n", cfgErr)
}
// if the config file wasn't provided, try to run using the default values + env vars
} else if cfgErr != nil {
// other errors while reading the config file
DieFmt("error reading configuration file: %v", cfgErr)
switch {
case cfgErr == nil:
logging.ContextUnavailable().
WithField("file", viper.ConfigFileUsed()).
Debug("loaded configuration from file")
case errors.As(cfgErr, &viper.ConfigFileNotFoundError{}):
if cfgFile != "" {
// specific message in case the file isn't found
DieFmt("config file not found, please run \"lakectl config\" to create one\n%s\n", cfgErr)
}
case cfgErr != nil:
DieFmt("error reading configuration file: %v", cfgErr)
}

err = viper.UnmarshalExact(&cfg, viper.DecodeHook(
mapstructure.ComposeDecodeHookFunc(
lakefsconfig.DecodeOnlyString,
mapstructure.StringToTimeDurationHookFunc())))
if err != nil {
DieFmt("error unmarshal configuration: %v", err)
}
err = viper.UnmarshalExact(&cfg, viper.DecodeHook(
mapstructure.ComposeDecodeHookFunc(
lakefsconfig.DecodeOnlyString,
mapstructure.StringToTimeDurationHookFunc())))
if err != nil {
DieFmt("error unmarshal configuration: %v", err)
}

if cmd.HasParent() {
// Don't send statistics for root command or if one of the excluding
var cmdName string
for curr := cmd; curr.HasParent(); curr = curr.Parent() {
if cmdName != "" {
cmdName = curr.Name() + "_" + cmdName
} else {
cmdName = curr.Name()
}
}
if !slices.Contains(excludeStatsCmds, cmdName) {
sendStats(cmd.Context(), getClient(), cmdName)
if cmd.HasParent() {
// Don't send statistics for root command or if one of the excluding
var cmdName string
for curr := cmd; curr.HasParent(); curr = curr.Parent() {
if cmdName != "" {
cmdName = curr.Name() + "_" + cmdName
} else {
cmdName = curr.Name()
}
}
},
if !slices.Contains(excludeStatsCmds, cmdName) {
sendStats(cmd.Context(), getClient(), cmdName)
}
}
}

// rootCmd represents the base command when called without any sub-commands
var rootCmd = &cobra.Command{
Use: "lakectl",
Short: "A cli tool to explore manage and work with lakeFS",
Long: `lakectl is a CLI tool allowing exploration and manipulation of a lakeFS environment`,
Run: func(cmd *cobra.Command, args []string) {
if !Must(cmd.Flags().GetBool("version")) {
if err := cmd.Help(); err != nil {
Expand Down Expand Up @@ -459,29 +462,47 @@ func getClient() *apigen.ClientWithResponses {
return client
}

func getBasename() string {
return strings.ToLower(filepath.Base(os.Args[0]))
}

// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
err := rootCmd.Execute()
ctx := context.Background()

var cmd *cobra.Command
baseName := getBasename()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: might want to extract that to a "setupLocalCommand" function

switch baseName {
case "git", "git.exe", "git-data", "git-data.exe":
cmd = localCmd
cmd.Use = baseName
cmd.SetContext(context.WithValue(ctx, lakectlLocalCommandNameKey, baseName))
default:
rootCmd.AddCommand(localCmd)
cmd = rootCmd
cmd.SetContext(context.WithValue(ctx, lakectlLocalCommandNameKey, "lakectl local"))
}
// make sure config is properly initialize
setupRootCommand(cmd)
cobra.OnInitialize(initConfig)
cmd.PersistentPreRun = rootPreRun
// run!
err := cmd.Execute()
if err != nil {
DieErr(err)
}
}

//nolint:gochecknoinits
func init() {
// Here you will define your flags and configuration settings.
// Cobra supports persistent flags, which, if defined here,
// will be global for your application.
cobra.OnInitialize(initConfig)
rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default is $HOME/.lakectl.yaml)")
rootCmd.PersistentFlags().BoolVar(&noColorRequested, "no-color", getEnvNoColor(), "don't use fancy output colors (default value can be set by NO_COLOR environment variable)")
rootCmd.PersistentFlags().StringVarP(&baseURI, "base-uri", "", os.Getenv("LAKECTL_BASE_URI"), "base URI used for lakeFS address parse")
rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "", "none", "set logging level")
rootCmd.PersistentFlags().StringVarP(&logFormat, "log-format", "", "", "set logging output format")
rootCmd.PersistentFlags().StringSliceVarP(&logOutputs, "log-output", "", []string{}, "set logging output(s)")
rootCmd.PersistentFlags().BoolVar(&verboseMode, "verbose", false, "run in verbose mode")
rootCmd.Flags().BoolP("version", "v", false, "version for lakectl")
func setupRootCommand(cmd *cobra.Command) {
cmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default is $HOME/.lakectl.yaml)")
cmd.PersistentFlags().BoolVar(&noColorRequested, "no-color", getEnvNoColor(), "don't use fancy output colors (default value can be set by NO_COLOR environment variable)")
cmd.PersistentFlags().StringVarP(&baseURI, "base-uri", "", os.Getenv("LAKECTL_BASE_URI"), "base URI used for lakeFS address parse")
cmd.PersistentFlags().StringVarP(&logLevel, "log-level", "", "none", "set logging level")
cmd.PersistentFlags().StringVarP(&logFormat, "log-format", "", "", "set logging output format")
cmd.PersistentFlags().StringSliceVarP(&logOutputs, "log-output", "", []string{}, "set logging output(s)")
cmd.PersistentFlags().BoolVar(&verboseMode, "verbose", false, "run in verbose mode")
cmd.Flags().BoolP("version", "v", false, "version for lakectl")
}

func getEnvNoColor() bool {
Expand Down
24 changes: 24 additions & 0 deletions docs/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -2507,6 +2507,30 @@ lakectl ingest --from <object store URI> --to <lakeFS path URI> [--dry-run] [fla



### lakectl install-git-plugin

set up `git data` (directory must exist and be in $PATH)

#### Synopsis
{:.no_toc}

Add a symlink to lakectl named `git-data`.
This allows calling `git data` and having it act as the `lakectl local` command
(as long as the symlink is within the executing users' $PATH environment variable

```
lakectl install-git-plugin <directory> [flags]
```

#### Options
{:.no_toc}

```
-h, --help help for install-git-plugin
```



### lakectl local

Sync local directories with lakeFS paths
Expand Down
Loading
Loading