From e650ecbeba940ab6d9bcd4b6229515ed5708e424 Mon Sep 17 00:00:00 2001 From: Arran Ubels Date: Tue, 8 Aug 2023 11:51:11 +1000 Subject: [PATCH] feat: findExecutable in user-supplied paths Implement `findExecutable` with strong caching. The main change from #3162 is that it now also accepts either a `file` or `file-list` for searching, so that one of several different options can be searched simultaneously for roughly equivalent implementations. See #3256 for the inspiration for this particular change. We *can* switch this to two template functions, but I think that the core implementation would best remain the way that it is. Resolves: #3141 Closes: #3162 Co-authored-by: Arran Ubels --- .../templates/functions/findExecutable.md | 39 ++++++++ assets/chezmoi.io/mkdocs.yml | 1 + internal/chezmoi/chezmoi_unix.go | 6 ++ internal/chezmoi/chezmoi_windows.go | 16 ++++ internal/chezmoi/findexecutable.go | 87 ++++++++++++++++++ .../chezmoi/findexecutable_darwin_test.go | 90 ++++++++++++++++++ internal/chezmoi/findexecutable_unix_test.go | 91 +++++++++++++++++++ .../chezmoi/findexecutable_windows_test.go | 69 ++++++++++++++ internal/cmd/config.go | 3 +- internal/cmd/templatefuncs.go | 27 ++++++ .../cmd/testdata/scripts/templatefuncs.txtar | 32 +++++++ internal/cmd/util.go | 18 ++++ internal/cmd/util_test.go | 46 ++++++++++ 13 files changed, 524 insertions(+), 1 deletion(-) create mode 100644 assets/chezmoi.io/docs/reference/templates/functions/findExecutable.md create mode 100644 internal/chezmoi/findexecutable.go create mode 100644 internal/chezmoi/findexecutable_darwin_test.go create mode 100644 internal/chezmoi/findexecutable_unix_test.go create mode 100644 internal/chezmoi/findexecutable_windows_test.go diff --git a/assets/chezmoi.io/docs/reference/templates/functions/findExecutable.md b/assets/chezmoi.io/docs/reference/templates/functions/findExecutable.md new file mode 100644 index 000000000000..4b9a392b3c12 --- /dev/null +++ b/assets/chezmoi.io/docs/reference/templates/functions/findExecutable.md @@ -0,0 +1,39 @@ +# `findExecutable` *file|file-list* *path-list* + +`findExecutable` searches for an executable named *file* in directories +identified by *path-list*. The search may be specified as a *file-list*, to find +the first matching executable in the first matching directory. The result will +be the executable file concatenated with the matching path. If an executable +*file* cannot be found in *path-list*, `findExecutable` returns an empty string. + +`findExecutable` is provided as an alternative to +[`lookPath`](/reference/templates/functions/lookPath) so that you can +interrogate the system PATH as it would be configured after `chezmoi apply`. +Like `lookPath`, `findExecutable` is not hermetic: its return value depends on +the state of the filesystem at the moment the template is executed. Exercise +caution when using it in your templates. + +The return value of the first successful call to `findExecutable` is cached, and +future calls to `findExecutable` with the same parameters (*file* and +*path-list* or *file-list* and *path-list*) will return this path. + +!!! info + + On Windows, the resulting path will contain the first found executable + extension as identified by the environment variable `%PathExt%`. + +!!! example + + ``` + {{ if findExecutable "rtx" (list "bin" "go/bin" ".cargo/bin" ".local/bin") }} + # $HOME/.cargo/bin/rtx exists and will probably be in $PATH after apply + {{ end }} + ``` + + With a list of possible executables: + + ``` + {{ if findExecutable (list "eza" "exa") (list "bin" "go/bin" ".cargo/bin" ".local/bin") }} + # $HOME/.cargo/bin/exa exists and will probably be in $PATH after apply + {{ end }} + ``` diff --git a/assets/chezmoi.io/mkdocs.yml b/assets/chezmoi.io/mkdocs.yml index 5ea502d744b0..5c09c59e7133 100644 --- a/assets/chezmoi.io/mkdocs.yml +++ b/assets/chezmoi.io/mkdocs.yml @@ -187,6 +187,7 @@ nav: - deleteValueAtPath: reference/templates/functions/deleteValueAtPath.md - encrypt: reference/templates/functions/encrypt.md - eqFold: reference/templates/functions/eqFold.md + - findExecutable: reference/templates/functions/findExecutable.md - fromIni: reference/templates/functions/fromIni.md - fromJsonc: reference/templates/functions/fromJsonc.md - fromToml: reference/templates/functions/fromToml.md diff --git a/internal/chezmoi/chezmoi_unix.go b/internal/chezmoi/chezmoi_unix.go index 80a3fe505562..6808bfe672a6 100644 --- a/internal/chezmoi/chezmoi_unix.go +++ b/internal/chezmoi/chezmoi_unix.go @@ -15,6 +15,12 @@ func init() { unix.Umask(int(Umask)) } +// findExecutableExtensions returns valid OS executable extensions, on unix it +// can be anything. +func findExecutableExtensions(path string) []string { + return []string{path} +} + // IsExecutable returns if fileInfo is executable. func IsExecutable(fileInfo fs.FileInfo) bool { return fileInfo.Mode().Perm()&0o111 != 0 diff --git a/internal/chezmoi/chezmoi_windows.go b/internal/chezmoi/chezmoi_windows.go index 6d0cd141e09e..c6fad0afcca0 100644 --- a/internal/chezmoi/chezmoi_windows.go +++ b/internal/chezmoi/chezmoi_windows.go @@ -13,6 +13,22 @@ const nativeLineEnding = "\r\n" var pathExts = strings.Split(os.Getenv("PATHEXT"), string(filepath.ListSeparator)) +// findExecutableExtensions returns valid OS executable extensions for the +// provided file if it does not already have an extension. The executable +// extensions are derived from %PathExt%. +func findExecutableExtensions(path string) []string { + cmdExt := filepath.Ext(path) + if cmdExt != "" { + return []string{path} + } + result := make([]string, len(pathExts)) + withoutSuffix := strings.TrimSuffix(path, cmdExt) + for i, ext := range pathExts { + result[i] = withoutSuffix + ext + } + return result +} + // IsExecutable checks if the file is a regular file and has an extension listed // in the PATHEXT environment variable as per // https://www.nextofwindows.com/what-is-pathext-environment-variable-in-windows. diff --git a/internal/chezmoi/findexecutable.go b/internal/chezmoi/findexecutable.go new file mode 100644 index 000000000000..dc2ba4161438 --- /dev/null +++ b/internal/chezmoi/findexecutable.go @@ -0,0 +1,87 @@ +package chezmoi + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" +) + +var ( + foundExecutableCacheMutex sync.Mutex + foundExecutableCache = make(map[string]string) +) + +// FindExecutable is like LookPath except that: +// +// - You can specify the needle as `string`, `[]string`, or `[]interface{}` +// (that converts to `[]string`). +// - You specify the haystack instead of relying on `$PATH`/`%PATH%`. +// +// This makes it useful for the resulting path of shell configurations +// managed by chezmoi. +func FindExecutable(fileOrFiles any, paths []string) (string, error) { + foundExecutableCacheMutex.Lock() + defer foundExecutableCacheMutex.Unlock() + + var files []string + + switch value := fileOrFiles.(type) { + case string: + files = []string{value} + case []string: + files = value + case []interface{}: + files = make([]string, 0, len(value)) + for _, e := range value { + if v, ok := e.(string); ok { + files = append(files, v) + } else { + return "", fmt.Errorf("%v: must be string or []string but is %T", value, value) + } + } + default: + return "", fmt.Errorf("%v: must be string or []string but is %T", value, value) + } + + key := strings.Join(files, "\x00") + "\x01" + strings.Join(paths, "\x00") + + if path, ok := foundExecutableCache[key]; ok { + return path, nil + } + + var candidates []string + + for _, file := range files { + candidates = append(candidates, findExecutableExtensions(file)...) + } + + // based on /usr/lib/go-1.20/src/os/exec/lp_unix.go:52 + for _, candidatePath := range paths { + if candidatePath == "" { + continue + } + + for _, candidate := range candidates { + path := filepath.Join(candidatePath, candidate) + + info, err := os.Stat(path) + if err != nil { + continue + } + + // isExecutable doesn't care if it's a directory + if info.Mode().IsDir() { + continue + } + + if IsExecutable(info) { + foundExecutableCache[key] = path + return path, nil + } + } + } + + return "", nil +} diff --git a/internal/chezmoi/findexecutable_darwin_test.go b/internal/chezmoi/findexecutable_darwin_test.go new file mode 100644 index 000000000000..9896d27e89fd --- /dev/null +++ b/internal/chezmoi/findexecutable_darwin_test.go @@ -0,0 +1,90 @@ +//go:build darwin + +package chezmoi + +import ( + "fmt" + "testing" +) + +func TestFindExecutableOneNeedle(t *testing.T) { + tests := []struct { + file string + paths []string + expected string + }{ + { + file: "sh", + paths: []string{"/usr/bin", "/bin"}, + expected: "/bin/sh", + }, + { + file: "sh", + paths: []string{"/bin", "/usr/bin"}, + expected: "/bin/sh", + }, + { + file: "chezmoish", + paths: []string{"/bin", "/usr/bin"}, + expected: "", + }, + } + + for _, test := range tests { + name := fmt.Sprintf("FindExecutable %v in %#v as %v", test.file, test.paths, test.expected) + t.Run(name, func(t *testing.T) { + actual, err := FindExecutable(test.file, test.paths) + if err != nil { + t.Errorf("FindExecutable() error = %v, expected = nil", err) + } + + if actual != test.expected { + t.Errorf("FindExecutable() actual = %v, expected = %v", actual, test.expected) + } + }) + } +} + +func TestFindExecutableMultipleNeedles(t *testing.T) { + tests := []struct { + files []string + paths []string + expected string + }{ + { + files: []string{"chezmoish", "sh"}, + paths: []string{"/usr/bin", "/bin"}, + expected: "/bin/sh", + }, + { + files: []string{"chezmoish", "sh"}, + paths: []string{"/bin", "/usr/bin"}, + expected: "/bin/sh", + }, + { + files: []string{"chezmoish", "chezvoush"}, + paths: []string{"/bin", "/usr/bin"}, + expected: "", + }, + } + + for _, test := range tests { + name := fmt.Sprintf( + "FindExecutable %#v in %#v as %v", + test.files, + test.paths, + test.expected, + ) + t.Run(name, func(t *testing.T) { + actual, err := FindExecutable(test.files, test.paths) + if err != nil { + t.Errorf("FindExecutable() error = %v, expected = nil", err) + return + } + + if actual != test.expected { + t.Errorf("FindExecutable() actual = %v, expected = %v", actual, test.expected) + } + }) + } +} diff --git a/internal/chezmoi/findexecutable_unix_test.go b/internal/chezmoi/findexecutable_unix_test.go new file mode 100644 index 000000000000..843706b1ab79 --- /dev/null +++ b/internal/chezmoi/findexecutable_unix_test.go @@ -0,0 +1,91 @@ +//go:build !windows && !darwin + +package chezmoi + +import ( + "fmt" + "testing" +) + +func TestFindExecutableOneNeedle(t *testing.T) { + tests := []struct { + file string + paths []string + expected string + }{ + { + file: "sh", + paths: []string{"/usr/bin", "/bin"}, + expected: "/usr/bin/sh", + }, + { + file: "sh", + paths: []string{"/bin", "/usr/bin"}, + expected: "/bin/sh", + }, + { + file: "chezmoish", + paths: []string{"/bin", "/usr/bin"}, + expected: "", + }, + } + + for _, test := range tests { + name := fmt.Sprintf("FindExecutable %v in %#v as %v", test.file, test.paths, test.expected) + t.Run(name, func(t *testing.T) { + actual, err := FindExecutable(test.file, test.paths) + if err != nil { + t.Errorf("FindExecutable() error = %v, expected = nil", err) + return + } + + if actual != test.expected { + t.Errorf("FindExecutable() actual = %v, expected = %v", actual, test.expected) + } + }) + } +} + +func TestFindExecutableMultipleNeedles(t *testing.T) { + tests := []struct { + files []string + paths []string + expected string + }{ + { + files: []string{"chezmoish", "sh"}, + paths: []string{"/usr/bin", "/bin"}, + expected: "/usr/bin/sh", + }, + { + files: []string{"chezmoish", "sh"}, + paths: []string{"/bin", "/usr/bin"}, + expected: "/bin/sh", + }, + { + files: []string{"chezmoish", "chezvoush"}, + paths: []string{"/bin", "/usr/bin"}, + expected: "", + }, + } + + for _, test := range tests { + name := fmt.Sprintf( + "FindExecutable %#v in %#v as %v", + test.files, + test.paths, + test.expected, + ) + t.Run(name, func(t *testing.T) { + actual, err := FindExecutable(test.files, test.paths) + if err != nil { + t.Errorf("FindExecutable() error = %v, expected = nil", err) + return + } + + if actual != test.expected { + t.Errorf("FindExecutable() actual = %v, expected = %v", actual, test.expected) + } + }) + } +} diff --git a/internal/chezmoi/findexecutable_windows_test.go b/internal/chezmoi/findexecutable_windows_test.go new file mode 100644 index 000000000000..d547008f530c --- /dev/null +++ b/internal/chezmoi/findexecutable_windows_test.go @@ -0,0 +1,69 @@ +//go:build windows + +package chezmoi + +import ( + "fmt" + "strings" + "testing" +) + +func TestFindExecutable(t *testing.T) { + tests := []struct { + file string + paths []string + expected string + }{ + { + file: "powershell.exe", + paths: []string{ + "c:\\windows\\system32", + "c:\\windows\\system64", + "C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0", + }, + expected: "C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", + }, + { + file: "powershell", + paths: []string{ + "c:\\windows\\system32", + "c:\\windows\\system64", + "C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0", + }, + expected: "C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", + }, + { + file: "weakshell.exe", + paths: []string{ + "c:\\windows\\system32", + "c:\\windows\\system64", + "C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0", + }, + expected: "", + }, + { + file: "weakshell", + paths: []string{ + "c:\\windows\\system32", + "c:\\windows\\system64", + "C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0", + }, + expected: "", + }, + } + + for _, test := range tests { + name := fmt.Sprintf("FindExecutable %v in %#v as %v", test.file, test.paths, test.expected) + t.Run(test.name, func(t *testing.T) { + actual, err := FindExecutable(test.file, test.paths) + if err != nil { + t.Errorf("FindExecutable() error = %v, expected = nil", err) + return + } + + if !strings.EqualFold(actual, test.expected) { + t.Errorf("FindExecutable() actual = %v, expected = %v", actual, test.expected) + } + }) + } +} diff --git a/internal/cmd/config.go b/internal/cmd/config.go index fc13d2358c3e..7955d642039e 100644 --- a/internal/cmd/config.go +++ b/internal/cmd/config.go @@ -408,6 +408,7 @@ func newConfig(options ...configOption) (*Config, error) { "ejsonDecryptWithKey": c.ejsonDecryptWithKeyTemplateFunc, "encrypt": c.encryptTemplateFunc, "eqFold": c.eqFoldTemplateFunc, + "findExecutable": c.findExecutableTemplateFunc, "fromIni": c.fromIniTemplateFunc, "fromJsonc": c.fromJsoncTemplateFunc, "fromToml": c.fromTomlTemplateFunc, @@ -450,8 +451,8 @@ func newConfig(options ...configOption) (*Config, error) { "output": c.outputTemplateFunc, "pass": c.passTemplateFunc, "passFields": c.passFieldsTemplateFunc, - "passRaw": c.passRawTemplateFunc, "passhole": c.passholeTemplateFunc, + "passRaw": c.passRawTemplateFunc, "pruneEmptyDicts": c.pruneEmptyDictsTemplateFunc, "quoteList": c.quoteListTemplateFunc, "rbw": c.rbwTemplateFunc, diff --git a/internal/cmd/templatefuncs.go b/internal/cmd/templatefuncs.go index ea9db44e8525..62795c8a84f4 100644 --- a/internal/cmd/templatefuncs.go +++ b/internal/cmd/templatefuncs.go @@ -133,6 +133,33 @@ func (c *Config) eqFoldTemplateFunc(first, second string, more ...string) bool { return false } +func (c *Config) findExecutableTemplateFunc(file, haystack any) string { + var paths []string + + switch value := haystack.(type) { + case []string: + paths = value + case []interface{}: + for _, e := range value { + v, ok := e.(string) + if !ok { + panic(errors.New("expected a list of string paths")) + } + + paths = append(paths, v) + } + default: + panic(errors.New("expected a list of string paths")) + } + + switch path, err := chezmoi.FindExecutable(file, paths); { + case err == nil: + return path + default: + panic(err) + } +} + func (c *Config) fromIniTemplateFunc(s string) map[string]any { file, err := ini.Load([]byte(s)) if err != nil { diff --git a/internal/cmd/testdata/scripts/templatefuncs.txtar b/internal/cmd/testdata/scripts/templatefuncs.txtar index 09e4de68945f..8ad2450c1051 100644 --- a/internal/cmd/testdata/scripts/templatefuncs.txtar +++ b/internal/cmd/testdata/scripts/templatefuncs.txtar @@ -78,6 +78,38 @@ stdout ^true$ exec chezmoi execute-template '{{ isExecutable "bin/not-executable" }}' stdout ^false$ +# test findExecutable template function to find in specified script varargs - success +[!windows] exec chezmoi execute-template '{{ findExecutable "echo" (list "/lib" "/bin" "/usr/bin") }}' +[!windows] stdout ^/bin/echo$ + +# test findExecutable template function to find in specified script varargs - success +[!windows] exec chezmoi execute-template '{{ findExecutable (list "chezmoish" "echo") (list "/lib" "/bin" "/usr/bin") }}' +[!windows] stdout ^/bin/echo$ + +# test findExecutable template function to find in specified script varargs - failure +[!windows] exec chezmoi execute-template '{{ findExecutable "echo" (list "/lib") }}' +[!windows] stdout ^$ + +# test findExecutable template function to find in specified script - success +[!windows] exec chezmoi execute-template '{{ findExecutable "echo" (list "/lib" "/bin" "/usr/bin") }}' +[!windows] stdout ^/bin/echo$ + +# test findExecutable template function to find in specified script - failure +[!windows] exec chezmoi execute-template '{{ findExecutable "echo" (list "/lib") }}' +[!windows] stdout ^$ + +# test findExecutable template function to find in specified script - success with extension +[windows] exec chezmoi execute-template '{{ findExecutable "git.exe" (list "c:\\windows\\system32" "c:\\windows\\system64" "C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0" "C:\\Program Files\\Git\\cmd") }}' +[windows] stdout 'git' + +# test findExecutable template function to find in specified script - success without extension +[windows] exec chezmoi execute-template '{{ findExecutable "git" (list "c:\\windows\\system32" "c:\\windows\\system64" "C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0" "C:\\Program Files\\Git\\cmd") }}' +[windows] stdout 'git' + +# test findExecutable template function to find in specified script - failure +[windows] exec chezmoi execute-template '{{ findExecutable "asdf" (list "c:\\windows\\system32" "c:\\windows\\system64" "C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0" "C:\\Program Files\\Git\\cmd") }}' +[windows] stdout '^$' + # test lookPath template function to find in PATH exec chezmoi execute-template '{{ lookPath "go" }}' stdout go$exe diff --git a/internal/cmd/util.go b/internal/cmd/util.go index 1243005b0e95..1274375a24cc 100644 --- a/internal/cmd/util.go +++ b/internal/cmd/util.go @@ -1,6 +1,7 @@ package cmd import ( + "reflect" "strings" "unicode" ) @@ -143,3 +144,20 @@ func upperSnakeCaseToCamelCaseMap[V any](m map[string]V) map[string]V { } return result } + +func flattenStringList(vpaths []any) []string { + var paths []string + for i := range vpaths { + switch path := vpaths[i].(type) { + case []string: + paths = append(paths, path...) + case string: + paths = append(paths, path) + case []any: + paths = append(paths, flattenStringList(path)...) + default: + panic("unknown type: " + reflect.TypeOf(path).String()) + } + } + return paths +} diff --git a/internal/cmd/util_test.go b/internal/cmd/util_test.go index 7e9841edbf19..80cb30478ea5 100644 --- a/internal/cmd/util_test.go +++ b/internal/cmd/util_test.go @@ -1,6 +1,8 @@ package cmd import ( + "reflect" + "strings" "testing" "github.com/alecthomas/assert/v2" @@ -148,3 +150,47 @@ func TestUpperSnakeCaseToCamelCaseMap(t *testing.T) { "id": "", }, actual) } + +func Test_flattenStringList(t *testing.T) { + tests := []struct { + name string + vpaths []any + want []string + }{ + { + name: "Nothing", + }, + { + name: "Just a string", + vpaths: []any{"1"}, + want: []string{"1"}, + }, + { + name: "Just a array of string", + vpaths: []any{[]string{"1", "2"}}, + want: []string{"1", "2"}, + }, + { + name: "Just a array of any containing string", + vpaths: []any{[]any{"1", "2"}}, + want: []string{"1", "2"}, + }, + { + name: "Just a array of any containing string", + vpaths: []any{[]any{"1", "2"}}, + want: []string{"1", "2"}, + }, + { + name: "Hybrid", + vpaths: []any{"0", []any{"1", "2"}, []any{[]string{"3", "4"}}, []any{[]any{"5", "6"}}}, + want: strings.Split("0123456", ""), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := flattenStringList(tt.vpaths); !reflect.DeepEqual(got, tt.want) { + t.Errorf("flattenStringList() = %v, want %v", got, tt.want) + } + }) + } +}