From 0ffac4c71c7fc3fd7526337e6b885ccbd63f4695 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Thu, 4 Aug 2022 15:13:51 +0200 Subject: [PATCH 01/38] feat: HostEnvironment --- pkg/container/docker_run.go | 3 +- pkg/container/executions_environment.go | 11 + pkg/container/host_environment.go | 505 ++++++++++++++++++ .../linux_container_environment_extensions.go | 71 +++ ...x_container_environment_extensions_test.go | 63 +++ pkg/container/util.go | 26 + pkg/container/util_openbsd_mips64.go | 17 + pkg/container/util_plan9.go | 17 + pkg/container/util_windows.go | 15 + pkg/lookpath/env.go | 18 + pkg/lookpath/error.go | 10 + pkg/lookpath/lp_js.go | 23 + pkg/lookpath/lp_plan9.go | 56 ++ pkg/lookpath/lp_unix.go | 59 ++ pkg/lookpath/lp_windows.go | 94 ++++ pkg/runner/action.go | 10 +- pkg/runner/container_mock_test.go | 1 + pkg/runner/expression.go | 26 +- pkg/runner/job_executor_test.go | 1 + pkg/runner/run_context.go | 142 +++-- pkg/runner/run_context_test.go | 4 +- pkg/runner/runner.go | 44 -- pkg/runner/runner_test.go | 58 -- pkg/runner/step.go | 9 +- pkg/runner/step_action_remote.go | 2 +- pkg/runner/step_docker.go | 7 +- pkg/runner/step_docker_test.go | 2 +- pkg/runner/step_run.go | 6 +- 28 files changed, 1131 insertions(+), 169 deletions(-) create mode 100644 pkg/container/executions_environment.go create mode 100644 pkg/container/host_environment.go create mode 100644 pkg/container/linux_container_environment_extensions.go create mode 100644 pkg/container/linux_container_environment_extensions_test.go create mode 100644 pkg/container/util.go create mode 100644 pkg/container/util_openbsd_mips64.go create mode 100644 pkg/container/util_plan9.go create mode 100644 pkg/container/util_windows.go create mode 100644 pkg/lookpath/env.go create mode 100644 pkg/lookpath/error.go create mode 100644 pkg/lookpath/lp_js.go create mode 100644 pkg/lookpath/lp_plan9.go create mode 100644 pkg/lookpath/lp_unix.go create mode 100644 pkg/lookpath/lp_windows.go diff --git a/pkg/container/docker_run.go b/pkg/container/docker_run.go index faf11a8e9a1..f74f805a453 100644 --- a/pkg/container/docker_run.go +++ b/pkg/container/docker_run.go @@ -80,7 +80,7 @@ type Container interface { } // NewContainer creates a reference to a container -func NewContainer(input *NewContainerInput) Container { +func NewContainer(input *NewContainerInput) ExecutionsEnvironment { cr := new(containerReference) cr.input = input return cr @@ -229,6 +229,7 @@ type containerReference struct { input *NewContainerInput UID int GID int + LinuxContainerEnvironmentExtensions } func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) { diff --git a/pkg/container/executions_environment.go b/pkg/container/executions_environment.go new file mode 100644 index 00000000000..73a1160ca6d --- /dev/null +++ b/pkg/container/executions_environment.go @@ -0,0 +1,11 @@ +package container + +type ExecutionsEnvironment interface { + Container + ToContainerPath(string) string + GetActPath() string + GetPathVariableName() string + DefaultPathVariable() string + JoinPathVariable(...string) string + GetRunnerContext() map[string]interface{} +} diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go new file mode 100644 index 00000000000..b649daf778e --- /dev/null +++ b/pkg/container/host_environment.go @@ -0,0 +1,505 @@ +package container + +import ( + "archive/tar" + "bufio" + "bytes" + "context" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "errors" + + "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/lookpath" + "golang.org/x/term" +) + +type HostEnvironment struct { + Path string + TmpDir string + ToolCache string + CleanUp func() + StdOut io.Writer +} + +func (e *HostEnvironment) Create(capAdd []string, capDrop []string) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Close() common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor { + return func(ctx context.Context) error { + for _, f := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(destPath, f.Name)), 0777); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(destPath, f.Name), []byte(f.Body), fs.FileMode(f.Mode)); err != nil { + return err + } + } + return nil + } +} + +func (e *HostEnvironment) CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor { + return func(ctx context.Context) error { + return filepath.Walk(srcPath, func(file string, fi os.FileInfo, err error) error { + if fi.Mode()&os.ModeSymlink != 0 { + lnk, err := os.Readlink(file) + if err != nil { + return err + } + relpath, err := filepath.Rel(srcPath, file) + if err != nil { + return err + } + fdestpath := filepath.Join(destPath, relpath) + if err := os.MkdirAll(filepath.Dir(fdestpath), 0777); err != nil { + return err + } + if err := os.Symlink(lnk, fdestpath); err != nil { + return err + } + } else if fi.Mode().IsRegular() { + relpath, err := filepath.Rel(srcPath, file) + if err != nil { + return err + } + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + fdestpath := filepath.Join(destPath, relpath) + if err := os.MkdirAll(filepath.Dir(fdestpath), 0777); err != nil { + return err + } + df, err := os.OpenFile(fdestpath, os.O_CREATE|os.O_WRONLY, fi.Mode()) + if err != nil { + return err + } + defer df.Close() + if _, err := io.Copy(df, f); err != nil { + return err + } + } + return nil + }) + } +} + +func fileCallbackfilecbk(srcPath string, tw *tar.Writer, file string, fi os.FileInfo, err error) error { + if fi.Mode()&os.ModeSymlink != 0 { + lnk, err := os.Readlink(file) + if err != nil { + return err + } + fih, err := tar.FileInfoHeader(fi, lnk) + if err != nil { + return err + } + fih.Name, err = filepath.Rel(srcPath, file) + if err != nil { + return err + } + if string(filepath.Separator) != "/" { + fih.Name = strings.ReplaceAll(fih.Name, string(filepath.Separator), "/") + } + if err := tw.WriteHeader(fih); err != nil { + return err + } + } else if fi.Mode().IsRegular() { + fih, err := tar.FileInfoHeader(fi, "") + if err != nil { + return err + } + fih.Name, err = filepath.Rel(srcPath, file) + if err != nil { + return err + } + if string(filepath.Separator) != "/" { + fih.Name = strings.ReplaceAll(fih.Name, string(filepath.Separator), "/") + } + if err := tw.WriteHeader(fih); err != nil { + return err + } + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + if _, err := io.Copy(tw, f); err != nil { + return err + } + } + return nil +} + +func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (rc io.ReadCloser, err error) { + buf := &bytes.Buffer{} + tw := tar.NewWriter(buf) + defer func() { + if err == nil { + err = tw.Close() + } + }() + srcPath = filepath.Clean(srcPath) + fi, err := os.Lstat(srcPath) + if err != nil { + return nil, err + } + filecbk := func(file string, fi os.FileInfo, err error) error { + return fileCallbackfilecbk(srcPath, tw, file, fi, err) + } + if fi.IsDir() { + if err := filepath.Walk(srcPath, filecbk); err != nil { + return nil, err + } + } else { + file := srcPath + srcPath = filepath.Dir(srcPath) + if err := filecbk(file, fi, nil); err != nil { + return nil, err + } + } + return io.NopCloser(buf), nil +} + +func (e *HostEnvironment) Pull(forcePull bool) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Start(attach bool) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +type ptyWriter struct { + Out io.Writer + AutoStop bool + dirtyLine bool +} + +func (w *ptyWriter) Write(buf []byte) (int, error) { + if w.AutoStop && len(buf) > 0 && buf[len(buf)-1] == 4 { + n, err := w.Out.Write(buf[:len(buf)-1]) + if err != nil { + return n, err + } + if w.dirtyLine || len(buf) > 1 && buf[len(buf)-2] != '\n' { + _, _ = w.Out.Write([]byte("\n")) + return n, io.EOF + } + return n, io.EOF + } + w.dirtyLine = strings.LastIndex(string(buf), "\n") < len(buf)-1 + return w.Out.Write(buf) +} + +type localEnv struct { + env map[string]string +} + +func (l *localEnv) Getenv(name string) string { + if runtime.GOOS == "windows" { + for k, v := range l.env { + if strings.EqualFold(name, k) { + return v + } + } + return "" + } + return l.env[name] +} + +func lookupPathHost(cmd string, env map[string]string, writer io.Writer) (string, error) { + f, err := lookpath.LookPath2(cmd, &localEnv{env: env}) + if err != nil { + err := "Cannot find: " + fmt.Sprint(cmd) + " in PATH" + if _, _err := writer.Write([]byte(err + "\n")); _err != nil { + return "", fmt.Errorf("%v: %w", err, _err) + } + return "", errors.New(err) + } + return f, nil +} + +func setupPty(cmd *exec.Cmd, cmdline string) (*os.File, *os.File, error) { + ppty, tty, err := openPty() + if err != nil { + return nil, nil, err + } + if term.IsTerminal(int(tty.Fd())) { + _, err := term.MakeRaw(int(tty.Fd())) + if err != nil { + ppty.Close() + tty.Close() + return nil, nil, err + } + } + cmd.Stdin = tty + cmd.Stdout = tty + cmd.Stderr = tty + cmd.SysProcAttr = getSysProcAttr(cmdline, true) + return ppty, tty, nil +} + +func writeKeepAlive(ppty io.Writer) { + c := 1 + var err error + for c == 1 && err == nil { + c, err = ppty.Write([]byte{4}) + <-time.After(time.Second) + } +} + +func copyPtyOutput(writer io.Writer, ppty io.Reader, finishLog context.CancelFunc) { + defer func() { + finishLog() + }() + if _, err := io.Copy(writer, ppty); err != nil { + return + } +} + +func (e *HostEnvironment) UpdateFromImageEnv(env *map[string]string) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func getEnvListFromMap(env map[string]string) []string { + envList := make([]string, 0) + for k, v := range env { + envList = append(envList, fmt.Sprintf("%s=%s", k, v)) + } + return envList +} + +func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, user, workdir string) error { + envList := getEnvListFromMap(env) + var wd string + if workdir != "" { + if strings.HasPrefix(workdir, "/") { + wd = workdir + } else { + wd = fmt.Sprintf("%s/%s", e.Path, workdir) + } + } else { + wd = e.Path + } + f, err := lookupPathHost(command[0], env, e.StdOut) + if err != nil { + return err + } + cmd := exec.CommandContext(ctx, f) + cmd.Path = f + cmd.Args = command + cmd.Stdin = nil + cmd.Stdout = e.StdOut + cmd.Env = envList + cmd.Stderr = e.StdOut + cmd.Dir = wd + cmd.SysProcAttr = getSysProcAttr(cmdline, false) + var ppty *os.File + var tty *os.File + defer func() { + if ppty != nil { + ppty.Close() + } + if tty != nil { + tty.Close() + } + }() + if true /* allocate Terminal */ { + var err error + ppty, tty, err = setupPty(cmd, cmdline) + if err != nil { + common.Logger(ctx).Debugf("Failed to setup Pty %v\n", err.Error()) + } + } + writer := &ptyWriter{Out: e.StdOut} + logctx, finishLog := context.WithCancel(context.Background()) + if ppty != nil { + go copyPtyOutput(writer, ppty, finishLog) + } else { + finishLog() + } + if ppty != nil { + go writeKeepAlive(ppty) + } + err = cmd.Run() + if err != nil { + return err + } + if tty != nil { + writer.AutoStop = true + if _, err := tty.Write([]byte("\x04")); err != nil { + common.Logger(ctx).Debug("Failed to write EOT") + } + } + <-logctx.Done() + + if ppty != nil { + ppty.Close() + ppty = nil + } + return err +} + +func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[string]string, user, workdir string) common.Executor { + return func(ctx context.Context) error { + if err := e.exec(ctx, command, "" /*cmdline*/, env, user, workdir); err != nil { + select { + case <-ctx.Done(): + return fmt.Errorf("this step has been cancelled: %w", err) + default: + return err + } + } + return nil + } +} + +var _singleLineEnvPattern *regexp.Regexp +var _mulitiLineEnvPattern *regexp.Regexp + +func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor { + if _singleLineEnvPattern == nil { + _singleLineEnvPattern = regexp.MustCompile("^([^=]+)=([^=]+)$") + _mulitiLineEnvPattern = regexp.MustCompile(`^([^<]+)<<(\w+)$`) + } + + localEnv := *env + return func(ctx context.Context) error { + envTar, err := e.GetContainerArchive(ctx, srcPath) + if err != nil { + return nil + } + defer envTar.Close() + reader := tar.NewReader(envTar) + _, err = reader.Next() + if err != nil && err != io.EOF { + return err + } + s := bufio.NewScanner(reader) + multiLineEnvKey := "" + multiLineEnvDelimiter := "" + multiLineEnvContent := "" + for s.Scan() { + line := s.Text() + if singleLineEnv := _singleLineEnvPattern.FindStringSubmatch(line); singleLineEnv != nil { + localEnv[singleLineEnv[1]] = singleLineEnv[2] + } + if line == multiLineEnvDelimiter { + localEnv[multiLineEnvKey] = multiLineEnvContent + multiLineEnvKey, multiLineEnvDelimiter, multiLineEnvContent = "", "", "" + } + if multiLineEnvKey != "" && multiLineEnvDelimiter != "" { + if multiLineEnvContent != "" { + multiLineEnvContent += "\n" + } + multiLineEnvContent += line + } + if mulitiLineEnvStart := _mulitiLineEnvPattern.FindStringSubmatch(line); mulitiLineEnvStart != nil { + multiLineEnvKey = mulitiLineEnvStart[1] + multiLineEnvDelimiter = mulitiLineEnvStart[2] + } + } + env = &localEnv + return nil + } +} + +func (e *HostEnvironment) UpdateFromPath(env *map[string]string) common.Executor { + localEnv := *env + return func(ctx context.Context) error { + pathTar, err := e.GetContainerArchive(ctx, localEnv["GITHUB_PATH"]) + if err != nil { + return err + } + defer pathTar.Close() + + reader := tar.NewReader(pathTar) + _, err = reader.Next() + if err != nil && err != io.EOF { + return err + } + s := bufio.NewScanner(reader) + for s.Scan() { + line := s.Text() + pathSep := string(filepath.ListSeparator) + localEnv[e.GetPathVariableName()] = fmt.Sprintf("%s%s%s", line, pathSep, localEnv["PATH"]) + } + + env = &localEnv + return nil + } +} + +func (e *HostEnvironment) Remove() common.Executor { + return func(ctx context.Context) error { + if e.CleanUp != nil { + e.CleanUp() + } + return os.RemoveAll(e.Path) + } +} + +func (*HostEnvironment) ToContainerPath(path string) string { + return path +} + +func (*HostEnvironment) GetActPath() string { + return "/var/run/act" +} + +func (*HostEnvironment) GetPathVariableName() string { + if runtime.GOOS == "plan9" { + return "path" + } + return "PATH" +} + +func (e *HostEnvironment) DefaultPathVariable() string { + v, _ := os.LookupEnv(e.GetPathVariableName()) + return v +} + +func (*HostEnvironment) JoinPathVariable(paths ...string) string { + return strings.Join(paths, string(filepath.ListSeparator)) +} + +func (h *HostEnvironment) GetRunnerContext() map[string]interface{} { + return map[string]interface{}{ + "os": runtime.GOOS, + "arch": runtime.GOARCH, + "temp": h.TmpDir, + "tool_cache": h.ToolCache, + } +} + +func (h *HostEnvironment) ReplaceLogWriter(stdout io.Writer, stderr io.Writer) (io.Writer, io.Writer) { + org := h.StdOut + h.StdOut = stdout + return org, org +} diff --git a/pkg/container/linux_container_environment_extensions.go b/pkg/container/linux_container_environment_extensions.go new file mode 100644 index 00000000000..5a9b8e1d317 --- /dev/null +++ b/pkg/container/linux_container_environment_extensions.go @@ -0,0 +1,71 @@ +package container + +import ( + "path/filepath" + "regexp" + "runtime" + "strings" + + log "github.com/sirupsen/logrus" +) + +type LinuxContainerEnvironmentExtensions struct { +} + +// Resolves the equivalent host path inside the container +// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject +// For use in docker volumes and binds +func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string { + if runtime.GOOS == "windows" && strings.Contains(path, "/") { + log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.") + return "" + } + + abspath, err := filepath.Abs(path) + if err != nil { + log.Error(err) + return "" + } + + // Test if the path is a windows path + windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`) + windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath) + + // Return as-is if no match + if windowsPathComponents == nil { + return abspath + } + + // Convert to WSL2-compatible path if it is a windows path + // NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows + // based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work. + driveLetter := strings.ToLower(windowsPathComponents[1]) + translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`) + // Should make something like /mnt/c/Users/person/My Folder/MyActProject + result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`) + return result +} + +func (*LinuxContainerEnvironmentExtensions) GetActPath() string { + return "/var/run/act" +} + +func (*LinuxContainerEnvironmentExtensions) GetPathVariableName() string { + return "PATH" +} + +func (*LinuxContainerEnvironmentExtensions) DefaultPathVariable() string { + return "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +} + +func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) string { + return strings.Join(paths, ":") +} + +func (*LinuxContainerEnvironmentExtensions) GetRunnerContext() map[string]interface{} { + return map[string]interface{}{ + "os": "Linux", + "temp": "/tmp", + "tool_cache": "/opt/hostedtoolcache", + } +} diff --git a/pkg/container/linux_container_environment_extensions_test.go b/pkg/container/linux_container_environment_extensions_test.go new file mode 100644 index 00000000000..3a0d366f0d7 --- /dev/null +++ b/pkg/container/linux_container_environment_extensions_test.go @@ -0,0 +1,63 @@ +package container + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func TestContainerPath(t *testing.T) { + type containerPathJob struct { + destinationPath string + sourcePath string + workDir string + } + + linuxcontainerext := &LinuxContainerEnvironmentExtensions{} + + if runtime.GOOS == "windows" { + cwd, err := os.Getwd() + if err != nil { + log.Error(err) + } + + rootDrive := os.Getenv("SystemDrive") + rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") + for _, v := range []containerPathJob{ + {"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""}, + {"/mnt/f/work/dir", `F:\work\dir`, ""}, + {"/mnt/c/windows/to/unix", "windows\\to\\unix", fmt.Sprintf("%s\\", rootDrive)}, + {fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", fmt.Sprintf("%s\\", rootDrive)}, + } { + if v.workDir != "" { + if err := os.Chdir(v.workDir); err != nil { + log.Error(err) + t.Fail() + } + } + + assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath)) + } + + if err := os.Chdir(cwd); err != nil { + log.Error(err) + } + } else { + cwd, err := os.Getwd() + if err != nil { + log.Error(err) + } + for _, v := range []containerPathJob{ + {"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""}, + {"/home/act", `/home/act/`, ""}, + {cwd, ".", ""}, + } { + assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath)) + } + } +} diff --git a/pkg/container/util.go b/pkg/container/util.go new file mode 100644 index 00000000000..6a47b9ab6c3 --- /dev/null +++ b/pkg/container/util.go @@ -0,0 +1,26 @@ +// +build !windows,!plan9,!openbsd !windows,!plan9,!mips64 + +package container + +import ( + "os" + "syscall" + + "github.com/creack/pty" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + if tty { + return &syscall.SysProcAttr{ + Setsid: true, + Setctty: true, + } + } + return &syscall.SysProcAttr{ + Setpgid: true, + } +} + +func openPty() (*os.File, *os.File, error) { + return pty.Open() +} diff --git a/pkg/container/util_openbsd_mips64.go b/pkg/container/util_openbsd_mips64.go new file mode 100644 index 00000000000..b991d694f1c --- /dev/null +++ b/pkg/container/util_openbsd_mips64.go @@ -0,0 +1,17 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{ + Setpgid: true, + } +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/pkg/container/util_plan9.go b/pkg/container/util_plan9.go new file mode 100644 index 00000000000..cd64b4abe19 --- /dev/null +++ b/pkg/container/util_plan9.go @@ -0,0 +1,17 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{ + Rfork: syscall.RFNOTEG, + } +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/pkg/container/util_windows.go b/pkg/container/util_windows.go new file mode 100644 index 00000000000..0a94c83abf6 --- /dev/null +++ b/pkg/container/util_windows.go @@ -0,0 +1,15 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{CmdLine: cmdLine, CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP} +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/pkg/lookpath/env.go b/pkg/lookpath/env.go new file mode 100644 index 00000000000..ce41e41aaba --- /dev/null +++ b/pkg/lookpath/env.go @@ -0,0 +1,18 @@ +package lookpath + +import "os" + +type Env interface { + Getenv(name string) string +} + +type defaultEnv struct { +} + +func (*defaultEnv) Getenv(name string) string { + return os.Getenv(name) +} + +func LookPath(file string) (string, error) { + return LookPath2(file, &defaultEnv{}) +} diff --git a/pkg/lookpath/error.go b/pkg/lookpath/error.go new file mode 100644 index 00000000000..eefed32f859 --- /dev/null +++ b/pkg/lookpath/error.go @@ -0,0 +1,10 @@ +package lookpath + +type Error struct { + Name string + Err error +} + +func (e *Error) Error() string { + return e.Err.Error() +} diff --git a/pkg/lookpath/lp_js.go b/pkg/lookpath/lp_js.go new file mode 100644 index 00000000000..09276245df1 --- /dev/null +++ b/pkg/lookpath/lp_js.go @@ -0,0 +1,23 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js,wasm + +package lookpath + +import ( + "errors" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $PATH") + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // Wasm can not execute processes, so act as if there are no executables at all. + return "", &Error{file, ErrNotFound} +} diff --git a/pkg/lookpath/lp_plan9.go b/pkg/lookpath/lp_plan9.go new file mode 100644 index 00000000000..a201b715608 --- /dev/null +++ b/pkg/lookpath/lp_plan9.go @@ -0,0 +1,56 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $path") + +func findExecutable(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if m := d.Mode(); !m.IsDir() && m&0111 != 0 { + return nil + } + return fs.ErrPermission +} + +// LookPath searches for an executable named file in the +// directories named by the path environment variable. +// If file begins with "/", "#", "./", or "../", it is tried +// directly and the path is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // skip the path lookup for these prefixes + skip := []string{"/", "#", "./", "../"} + + for _, p := range skip { + if strings.HasPrefix(file, p) { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + } + + path := lenv.Getenv("path") + for _, dir := range filepath.SplitList(path) { + path := filepath.Join(dir, file) + if err := findExecutable(path); err == nil { + return path, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/pkg/lookpath/lp_unix.go b/pkg/lookpath/lp_unix.go new file mode 100644 index 00000000000..d672cc3aa18 --- /dev/null +++ b/pkg/lookpath/lp_unix.go @@ -0,0 +1,59 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $PATH") + +func findExecutable(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if m := d.Mode(); !m.IsDir() && m&0111 != 0 { + return nil + } + return fs.ErrPermission +} + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // NOTE(rsc): I wish we could use the Plan 9 behavior here + // (only bypass the path if file begins with / or ./ or ../) + // but that would not match all the Unix shells. + + if strings.Contains(file, "/") { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + path := lenv.Getenv("PATH") + for _, dir := range filepath.SplitList(path) { + if dir == "" { + // Unix shell semantics: path element "" means "." + dir = "." + } + path := filepath.Join(dir, file) + if err := findExecutable(path); err == nil { + return path, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/pkg/lookpath/lp_windows.go b/pkg/lookpath/lp_windows.go new file mode 100644 index 00000000000..48204a79390 --- /dev/null +++ b/pkg/lookpath/lp_windows.go @@ -0,0 +1,94 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in %PATH%") + +func chkStat(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if d.IsDir() { + return fs.ErrPermission + } + return nil +} + +func hasExt(file string) bool { + i := strings.LastIndex(file, ".") + if i < 0 { + return false + } + return strings.LastIndexAny(file, `:\/`) < i +} + +func findExecutable(file string, exts []string) (string, error) { + if len(exts) == 0 { + return file, chkStat(file) + } + if hasExt(file) { + if chkStat(file) == nil { + return file, nil + } + } + for _, e := range exts { + if f := file + e; chkStat(f) == nil { + return f, nil + } + } + return "", fs.ErrNotExist +} + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// LookPath also uses PATHEXT environment variable to match +// a suitable candidate. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + var exts []string + x := lenv.Getenv(`PATHEXT`) + if x != "" { + for _, e := range strings.Split(strings.ToLower(x), `;`) { + if e == "" { + continue + } + if e[0] != '.' { + e = "." + e + } + exts = append(exts, e) + } + } else { + exts = []string{".com", ".exe", ".bat", ".cmd"} + } + + if strings.ContainsAny(file, `:\/`) { + if f, err := findExecutable(file, exts); err == nil { + return f, nil + } else { + return "", &Error{file, err} + } + } + if f, err := findExecutable(filepath.Join(".", file), exts); err == nil { + return f, nil + } + path := lenv.Getenv("path") + for _, dir := range filepath.SplitList(path) { + if f, err := findExecutable(filepath.Join(dir, file), exts); err == nil { + return f, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/pkg/runner/action.go b/pkg/runner/action.go index 72970f4e132..05773ac39c5 100644 --- a/pkg/runner/action.go +++ b/pkg/runner/action.go @@ -3,6 +3,7 @@ package runner import ( "context" "embed" + "errors" "fmt" "io" "io/fs" @@ -285,6 +286,9 @@ func execAsDocker(ctx context.Context, step actionStep, actionName string, based } } stepContainer := newStepContainer(ctx, step, image, cmd, entrypoint) + if stepContainer == nil { + return errors.New("Failed to create step container") + } return common.NewPipelineExecutor( prepImage, stepContainer.Pull(rc.Config.ForcePull), @@ -353,7 +357,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd []string stepContainer := container.NewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: rc.JobContainer.ToContainerPath(rc.Config.Workdir), Image: image, Username: rc.Config.Secrets["DOCKER_USERNAME"], Password: rc.Config.Secrets["DOCKER_PASSWORD"], @@ -420,11 +424,11 @@ func getContainerActionPaths(step *model.Step, actionDir string, rc *RunContext) containerActionDir := "." if step.Type() != model.StepTypeUsesActionRemote { actionName = getOsSafeRelativePath(actionDir, rc.Config.Workdir) - containerActionDir = rc.Config.ContainerWorkdir() + "/" + actionName + containerActionDir = rc.JobContainer.ToContainerPath(rc.Config.Workdir) + "/" + actionName actionName = "./" + actionName } else if step.Type() == model.StepTypeUsesActionRemote { actionName = getOsSafeRelativePath(actionDir, rc.ActionCacheDir()) - containerActionDir = ActPath + "/actions/" + actionName + containerActionDir = rc.JobContainer.GetActPath() + "/actions/" + actionName } if actionName == "" { diff --git a/pkg/runner/container_mock_test.go b/pkg/runner/container_mock_test.go index a336d404a02..0de0781586f 100644 --- a/pkg/runner/container_mock_test.go +++ b/pkg/runner/container_mock_test.go @@ -11,6 +11,7 @@ import ( type containerMock struct { mock.Mock container.Container + container.LinuxContainerEnvironmentExtensions } func (cm *containerMock) Create(capAdd []string, capDrop []string) common.Executor { diff --git a/pkg/runner/expression.go b/pkg/runner/expression.go index ab54e5e3e19..bfdfb509aec 100644 --- a/pkg/runner/expression.go +++ b/pkg/runner/expression.go @@ -44,18 +44,16 @@ func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEval Job: rc.getJobContext(), // todo: should be unavailable // but required to interpolate/evaluate the step outputs on the job - Steps: rc.getStepsContext(), - Runner: map[string]interface{}{ - "os": "Linux", - "temp": "/tmp", - "tool_cache": "/opt/hostedtoolcache", - }, + Steps: rc.getStepsContext(), Secrets: rc.Config.Secrets, Strategy: strategy, Matrix: rc.Matrix, Needs: using, Inputs: rc.Inputs, } + if rc.JobContainer != nil { + ee.Runner = rc.JobContainer.GetRunnerContext() + } return expressionEvaluator{ interpreter: exprparser.NewInterpeter(ee, exprparser.Config{ Run: rc.Run, @@ -86,15 +84,10 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) } ee := &exprparser.EvaluationEnvironment{ - Github: rc.getGithubContext(ctx), - Env: *step.getEnv(), - Job: rc.getJobContext(), - Steps: rc.getStepsContext(), - Runner: map[string]interface{}{ - "os": "Linux", - "temp": "/tmp", - "tool_cache": "/opt/hostedtoolcache", - }, + Github: rc.getGithubContext(ctx), + Env: *step.getEnv(), + Job: rc.getJobContext(), + Steps: rc.getStepsContext(), Secrets: rc.Config.Secrets, Strategy: strategy, Matrix: rc.Matrix, @@ -103,6 +96,9 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) // but required to interpolate/evaluate the inputs in actions/composite Inputs: rc.Inputs, } + if rc.JobContainer != nil { + ee.Runner = rc.JobContainer.GetRunnerContext() + } return expressionEvaluator{ interpreter: exprparser.NewInterpeter(ee, exprparser.Config{ Run: rc.Run, diff --git a/pkg/runner/job_executor_test.go b/pkg/runner/job_executor_test.go index 8299f63f587..edcbfc1b9e6 100644 --- a/pkg/runner/job_executor_test.go +++ b/pkg/runner/job_executor_test.go @@ -79,6 +79,7 @@ func (jim *jobInfoMock) result(result string) { type jobContainerMock struct { container.Container + container.LinuxContainerEnvironmentExtensions } func (jcm *jobContainerMock) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) { diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index acbff9b1d77..93fda6b6ba6 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -2,7 +2,10 @@ package runner import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -23,29 +26,28 @@ import ( "github.com/nektos/act/pkg/model" ) -const ActPath string = "/var/run/act" - // RunContext contains info about current job type RunContext struct { - Name string - Config *Config - Matrix map[string]interface{} - Run *model.Run - EventJSON string - Env map[string]string - ExtraPath []string - CurrentStep string - StepResults map[string]*model.StepResult - ExprEval ExpressionEvaluator - JobContainer container.Container - OutputMappings map[MappableOutput]MappableOutput - JobName string - ActionPath string - ActionRef string - ActionRepository string - Inputs map[string]interface{} - Parent *RunContext - Masks []string + Name string + Config *Config + Matrix map[string]interface{} + Run *model.Run + EventJSON string + Env map[string]string + ExtraPath []string + CurrentStep string + StepResults map[string]*model.StepResult + ExprEval ExpressionEvaluator + JobContainer container.ExecutionsEnvironment + OutputMappings map[MappableOutput]MappableOutput + JobName string + ActionPath string + ActionRef string + ActionRepository string + Inputs map[string]interface{} + Parent *RunContext + Masks []string + cleanUpJobContainer common.Executor } func (rc *RunContext) AddMask(mask string) { @@ -86,9 +88,11 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { fmt.Sprintf("%s:%s", rc.Config.ContainerDaemonSocket, "/var/run/docker.sock"), } + ext := container.LinuxContainerEnvironmentExtensions{} + mounts := map[string]string{ "act-toolcache": "/toolcache", - name + "-env": ActPath, + name + "-env": ext.GetActPath(), } if job := rc.Run.Job(); job != nil { @@ -114,9 +118,9 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { if selinux.GetEnabled() { bindModifiers = ":z" } - binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, rc.Config.ContainerWorkdir(), bindModifiers)) + binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, ext.ToContainerPath(rc.Config.Workdir), bindModifiers)) } else { - mounts[name] = rc.Config.ContainerWorkdir() + mounts[name] = ext.ToContainerPath(rc.Config.Workdir) } return binds, mounts @@ -126,7 +130,6 @@ func (rc *RunContext) startJobContainer() common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) image := rc.platformImage(ctx) - hostname := rc.hostname(ctx) rawLogger := logger.WithField("raw_output", true) logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { if rc.Config.LogOutput { @@ -136,6 +139,62 @@ func (rc *RunContext) startJobContainer() common.Executor { } return true }) + if image == "-self-hosted" { + cacheDir := rc.ActionCacheDir() + randBytes := make([]byte, 16) + _, _ = rand.Read(randBytes) + miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes)) + actPath := filepath.Join(miscpath, "act") + if err := os.MkdirAll(actPath, 0777); err != nil { + return err + } + path := filepath.Join(miscpath, "hostexecutor") + if err := os.MkdirAll(path, 0777); err != nil { + return err + } + runnerTmp := filepath.Join(miscpath, "tmp") + if err := os.MkdirAll(runnerTmp, 0777); err != nil { + return err + } + toolCache := filepath.Join(cacheDir, "tool_cache") + rc.JobContainer = &container.HostEnvironment{ + Path: path, + TmpDir: runnerTmp, + ToolCache: toolCache, + CleanUp: func() { + os.RemoveAll(miscpath) + }, + StdOut: logWriter, + } + rc.cleanUpJobContainer = rc.JobContainer.Remove() + rc.Env["RUNNER_TOOL_CACHE"] = toolCache + rc.Env["RUNNER_OS"] = runtime.GOOS + rc.Env["RUNNER_ARCH"] = runtime.GOARCH + rc.Env["RUNNER_TEMP"] = runnerTmp + for _, env := range os.Environ() { + i := strings.Index(env, "=") + if i > 0 { + rc.Env[env[0:i]] = env[i+1:] + } + } + + return common.NewPipelineExecutor( + rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ + Name: "workflow/event.json", + Mode: 0644, + Body: rc.EventJSON, + }, &container.FileEntry{ + Name: "workflow/envs.txt", + Mode: 0666, + Body: "", + }, &container.FileEntry{ + Name: "workflow/paths.txt", + Mode: 0666, + Body: "", + }), + )(ctx) + } + hostname := rc.hostname(ctx) username, password, err := rc.handleCredentials(ctx) if err != nil { @@ -151,12 +210,22 @@ func (rc *RunContext) startJobContainer() common.Executor { envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_OS", "Linux")) envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp")) + ext := container.LinuxContainerEnvironmentExtensions{} binds, mounts := rc.GetBindsAndMounts() + rc.cleanUpJobContainer = func(ctx context.Context) error { + if rc.JobContainer != nil && !rc.Config.ReuseContainers { + return rc.JobContainer.Remove(). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false))(ctx) + } + return nil + } + rc.JobContainer = container.NewContainer(&container.NewContainerInput{ Cmd: nil, Entrypoint: []string{"/usr/bin/tail", "-f", "/dev/null"}, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: ext.ToContainerPath(rc.Config.Workdir), Image: image, Username: username, Password: password, @@ -172,6 +241,9 @@ func (rc *RunContext) startJobContainer() common.Executor { Platform: rc.Config.ContainerArchitecture, Hostname: hostname, }) + if rc.JobContainer == nil { + return errors.New("Failed to create job container") + } return common.NewPipelineExecutor( rc.JobContainer.Pull(rc.Config.ForcePull), @@ -180,7 +252,7 @@ func (rc *RunContext) startJobContainer() common.Executor { rc.JobContainer.Start(false), rc.JobContainer.UpdateFromImageEnv(&rc.Env), rc.JobContainer.UpdateFromEnv("/etc/environment", &rc.Env), - rc.JobContainer.Copy(ActPath+"/", &container.FileEntry{ + rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ Name: "workflow/event.json", Mode: 0644, Body: rc.EventJSON, @@ -206,10 +278,8 @@ func (rc *RunContext) execJobContainer(cmd []string, env map[string]string, user // stopJobContainer removes the job container (if it exists) and its volume (if it exists) if !rc.Config.ReuseContainers func (rc *RunContext) stopJobContainer() common.Executor { return func(ctx context.Context) error { - if rc.JobContainer != nil && !rc.Config.ReuseContainers { - return rc.JobContainer.Remove(). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false))(ctx) + if rc.cleanUpJobContainer != nil && !rc.Config.ReuseContainers { + return rc.cleanUpJobContainer(ctx) } return nil } @@ -427,13 +497,11 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext logger := common.Logger(ctx) ghc := &model.GithubContext{ Event: make(map[string]interface{}), - EventPath: ActPath + "/workflow/event.json", Workflow: rc.Run.Workflow.Name, RunID: rc.Config.Env["GITHUB_RUN_ID"], RunNumber: rc.Config.Env["GITHUB_RUN_NUMBER"], Actor: rc.Config.Actor, EventName: rc.Config.EventName, - Workspace: rc.Config.ContainerWorkdir(), Action: rc.CurrentStep, Token: rc.Config.Token, ActionPath: rc.ActionPath, @@ -444,6 +512,10 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext RunnerPerflog: rc.Config.Env["RUNNER_PERFLOG"], RunnerTrackingID: rc.Config.Env["RUNNER_TRACKING_ID"], } + if rc.JobContainer != nil { + ghc.EventPath = rc.JobContainer.GetActPath() + "/workflow/event.json" + ghc.Workspace = rc.JobContainer.ToContainerPath(rc.Config.Workdir) + } if ghc.RunID == "" { ghc.RunID = "1" @@ -559,8 +631,8 @@ func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}) func (rc *RunContext) withGithubEnv(ctx context.Context, env map[string]string) map[string]string { github := rc.getGithubContext(ctx) env["CI"] = "true" - env["GITHUB_ENV"] = ActPath + "/workflow/envs.txt" - env["GITHUB_PATH"] = ActPath + "/workflow/paths.txt" + env["GITHUB_ENV"] = rc.JobContainer.GetActPath() + "/workflow/envs.txt" + env["GITHUB_PATH"] = rc.JobContainer.GetActPath() + "/workflow/paths.txt" env["GITHUB_WORKFLOW"] = github.Workflow env["GITHUB_RUN_ID"] = github.RunID env["GITHUB_RUN_NUMBER"] = github.RunNumber diff --git a/pkg/runner/run_context_test.go b/pkg/runner/run_context_test.go index 5980a8c6d63..03b30c77c58 100644 --- a/pkg/runner/run_context_test.go +++ b/pkg/runner/run_context_test.go @@ -385,14 +385,14 @@ func TestGetGitHubContext(t *testing.T) { } assert.Equal(t, ghc.RunID, "1") - assert.Equal(t, ghc.Workspace, rc.Config.containerPath(cwd)) + assert.Equal(t, ghc.Workspace, rc.JobContainer.ToContainerPath(cwd)) assert.Equal(t, ghc.RunNumber, "1") assert.Equal(t, ghc.RetentionDays, "0") assert.Equal(t, ghc.Actor, actor) assert.Equal(t, ghc.Repository, repo) assert.Equal(t, ghc.RepositoryOwner, owner) assert.Equal(t, ghc.RunnerPerflog, "/dev/null") - assert.Equal(t, ghc.EventPath, ActPath+"/workflow/event.json") + assert.Equal(t, ghc.EventPath, rc.JobContainer.GetActPath()+"/workflow/event.json") assert.Equal(t, ghc.Token, rc.Config.Secrets["GITHUB_TOKEN"]) } diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index aa66af7a252..167e41c550b 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -4,10 +4,6 @@ import ( "context" "fmt" "io/ioutil" - "path/filepath" - "regexp" - "runtime" - "strings" "time" log "github.com/sirupsen/logrus" @@ -57,46 +53,6 @@ type Config struct { ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub. } -// Resolves the equivalent host path inside the container -// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject -// For use in docker volumes and binds -func (config *Config) containerPath(path string) string { - if runtime.GOOS == "windows" && strings.Contains(path, "/") { - log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.") - return "" - } - - abspath, err := filepath.Abs(path) - if err != nil { - log.Error(err) - return "" - } - - // Test if the path is a windows path - windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`) - windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath) - - // Return as-is if no match - if windowsPathComponents == nil { - return abspath - } - - // Convert to WSL2-compatible path if it is a windows path - // NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows - // based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work. - driveLetter := strings.ToLower(windowsPathComponents[1]) - translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`) - // Should make something like /mnt/c/Users/person/My Folder/MyActProject - result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`) - return result -} - -// Resolves the equivalent host path inside the container -// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject -func (config *Config) ContainerWorkdir() string { - return config.containerPath(config.Workdir) -} - type runnerImpl struct { config *Config eventJSON string diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index b57b4ab992d..cf63ea7b160 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "path/filepath" - "runtime" "strings" "testing" @@ -306,60 +305,3 @@ func TestRunEventPullRequest(t *testing.T) { tjfi.runTest(context.Background(), t, &Config{EventPath: filepath.Join(workdir, workflowPath, "event.json")}) } - -func TestContainerPath(t *testing.T) { - type containerPathJob struct { - destinationPath string - sourcePath string - workDir string - } - - if runtime.GOOS == "windows" { - cwd, err := os.Getwd() - if err != nil { - log.Error(err) - } - - rootDrive := os.Getenv("SystemDrive") - rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") - for _, v := range []containerPathJob{ - {"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""}, - {"/mnt/f/work/dir", `F:\work\dir`, ""}, - {"/mnt/c/windows/to/unix", "windows\\to\\unix", fmt.Sprintf("%s\\", rootDrive)}, - {fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", fmt.Sprintf("%s\\", rootDrive)}, - } { - if v.workDir != "" { - if err := os.Chdir(v.workDir); err != nil { - log.Error(err) - t.Fail() - } - } - - runnerConfig := &Config{ - Workdir: v.sourcePath, - } - - assert.Equal(t, v.destinationPath, runnerConfig.containerPath(runnerConfig.Workdir)) - } - - if err := os.Chdir(cwd); err != nil { - log.Error(err) - } - } else { - cwd, err := os.Getwd() - if err != nil { - log.Error(err) - } - for _, v := range []containerPathJob{ - {"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""}, - {"/home/act", `/home/act/`, ""}, - {cwd, ".", ""}, - } { - runnerConfig := &Config{ - Workdir: v.sourcePath, - } - - assert.Equal(t, v.destinationPath, runnerConfig.containerPath(runnerConfig.Workdir)) - } - } -} diff --git a/pkg/runner/step.go b/pkg/runner/step.go index cf403b6f6e5..36dcb8df23f 100644 --- a/pkg/runner/step.go +++ b/pkg/runner/step.go @@ -153,13 +153,12 @@ func mergeEnv(ctx context.Context, step step) { mergeIntoMap(env, rc.GetEnv()) } - if (*env)["PATH"] == "" { - (*env)["PATH"] = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` + path := rc.JobContainer.GetPathVariableName() + if (*env)[path] == "" { + (*env)[path] = rc.JobContainer.DefaultPathVariable() } if rc.ExtraPath != nil && len(rc.ExtraPath) > 0 { - p := (*env)["PATH"] - (*env)["PATH"] = strings.Join(rc.ExtraPath, `:`) - (*env)["PATH"] += `:` + p + (*env)[path] = rc.JobContainer.JoinPathVariable(append(rc.ExtraPath, (*env)[path])...) } mergeIntoMap(env, rc.withGithubEnv(ctx, *env)) diff --git a/pkg/runner/step_action_remote.go b/pkg/runner/step_action_remote.go index b0f23c12fcd..6c392d1af36 100644 --- a/pkg/runner/step_action_remote.go +++ b/pkg/runner/step_action_remote.go @@ -123,7 +123,7 @@ func (sar *stepActionRemote) main() common.Executor { return nil } eval := sar.RunContext.NewExpressionEvaluator(ctx) - copyToPath := path.Join(sar.RunContext.Config.ContainerWorkdir(), eval.Interpolate(ctx, sar.Step.With["path"])) + copyToPath := path.Join(sar.RunContext.JobContainer.ToContainerPath(sar.RunContext.Config.Workdir), eval.Interpolate(ctx, sar.Step.With["path"])) return sar.RunContext.JobContainer.CopyDir(copyToPath, sar.RunContext.Config.Workdir+string(filepath.Separator)+".", sar.RunContext.Config.UseGitIgnore)(ctx) } diff --git a/pkg/runner/step_docker.go b/pkg/runner/step_docker.go index 0130b833495..4117182e2af 100644 --- a/pkg/runner/step_docker.go +++ b/pkg/runner/step_docker.go @@ -2,6 +2,7 @@ package runner import ( "context" + "errors" "fmt" "strings" @@ -69,7 +70,9 @@ func (sd *stepDocker) runUsesContainer() common.Executor { } stepContainer := sd.newStepContainer(ctx, image, cmd, entrypoint) - + if stepContainer == nil { + return errors.New("Failed to create step container") + } return common.NewPipelineExecutor( stepContainer.Pull(rc.Config.ForcePull), stepContainer.Remove().IfBool(!rc.Config.ReuseContainers), @@ -111,7 +114,7 @@ func (sd *stepDocker) newStepContainer(ctx context.Context, image string, cmd [] stepContainer := ContainerNewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: rc.JobContainer.ToContainerPath(rc.Config.Workdir), Image: image, Username: rc.Config.Secrets["DOCKER_USERNAME"], Password: rc.Config.Secrets["DOCKER_PASSWORD"], diff --git a/pkg/runner/step_docker_test.go b/pkg/runner/step_docker_test.go index c7482718e29..c6794b472e2 100644 --- a/pkg/runner/step_docker_test.go +++ b/pkg/runner/step_docker_test.go @@ -17,7 +17,7 @@ func TestStepDockerMain(t *testing.T) { // mock the new container call origContainerNewContainer := ContainerNewContainer - ContainerNewContainer = func(containerInput *container.NewContainerInput) container.Container { + ContainerNewContainer = func(containerInput *container.NewContainerInput) container.ExecutionsEnvironment { input = containerInput return cm } diff --git a/pkg/runner/step_run.go b/pkg/runner/step_run.go index 87c9516a4e1..5fd9b757aee 100644 --- a/pkg/runner/step_run.go +++ b/pkg/runner/step_run.go @@ -64,7 +64,8 @@ func (sr *stepRun) setupShellCommandExecutor() common.Executor { return err } - return sr.RunContext.JobContainer.Copy(ActPath, &container.FileEntry{ + rc := sr.getRunContext() + return sr.RunContext.JobContainer.Copy(rc.JobContainer.GetActPath(), &container.FileEntry{ Name: scriptName, Mode: 0755, Body: script, @@ -124,7 +125,8 @@ func (sr *stepRun) setupShellCommand(ctx context.Context) (name, script string, logger.Debugf("Wrote add-mask command to '%s'", name) } - scriptPath := fmt.Sprintf("%s/%s", ActPath, name) + rc := sr.getRunContext() + scriptPath := fmt.Sprintf("%s/%s", rc.JobContainer.GetActPath(), name) sr.cmd, err = shellquote.Split(strings.Replace(scCmd, `{0}`, scriptPath, 1)) return name, script, err From 7f3b9b0a02987963121331a6a7d81ff69e13192e Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Thu, 4 Aug 2022 15:18:24 +0200 Subject: [PATCH 02/38] fix: lint --- pkg/container/host_environment.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index b649daf778e..cfdd6546ddf 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -489,17 +489,17 @@ func (*HostEnvironment) JoinPathVariable(paths ...string) string { return strings.Join(paths, string(filepath.ListSeparator)) } -func (h *HostEnvironment) GetRunnerContext() map[string]interface{} { +func (e *HostEnvironment) GetRunnerContext() map[string]interface{} { return map[string]interface{}{ "os": runtime.GOOS, "arch": runtime.GOARCH, - "temp": h.TmpDir, - "tool_cache": h.ToolCache, + "temp": e.TmpDir, + "tool_cache": e.ToolCache, } } -func (h *HostEnvironment) ReplaceLogWriter(stdout io.Writer, stderr io.Writer) (io.Writer, io.Writer) { - org := h.StdOut - h.StdOut = stdout +func (e *HostEnvironment) ReplaceLogWriter(stdout io.Writer, stderr io.Writer) (io.Writer, io.Writer) { + org := e.StdOut + e.StdOut = stdout return org, org } From 5cbbeb9bd5afe1f83aa90710c272afc22ccfb59b Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Thu, 4 Aug 2022 15:54:23 +0200 Subject: [PATCH 03/38] fixes and update code --- pkg/container/file_collector.go | 23 +++++ pkg/container/host_environment.go | 159 ++++++++++++------------------ pkg/runner/run_context.go | 3 +- 3 files changed, 86 insertions(+), 99 deletions(-) diff --git a/pkg/container/file_collector.go b/pkg/container/file_collector.go index e97cadaedc5..2f86579cce1 100644 --- a/pkg/container/file_collector.go +++ b/pkg/container/file_collector.go @@ -59,6 +59,29 @@ func (tc tarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, return nil } +type copyCollector struct { + DstDir string +} + +func (cc *copyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { + fdestpath := filepath.Join(cc.DstDir, fpath) + if err := os.MkdirAll(filepath.Dir(fdestpath), 0777); err != nil { + return err + } + if f == nil { + return os.Symlink(linkName, fdestpath) + } + df, err := os.OpenFile(fdestpath, os.O_CREATE|os.O_WRONLY, fi.Mode()) + if err != nil { + return err + } + defer df.Close() + if _, err := io.Copy(df, f); err != nil { + return err + } + return nil +} + type fileCollector struct { Ignorer gitignore.Matcher SrcPath string diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index cfdd6546ddf..b4a7da6995c 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -18,6 +18,9 @@ import ( "errors" + "github.com/go-git/go-billy/v5/helper/polyfill" + "github.com/go-git/go-billy/v5/osfs" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/lookpath" "golang.org/x/term" @@ -27,6 +30,7 @@ type HostEnvironment struct { Path string TmpDir string ToolCache string + Workdir string CleanUp func() StdOut io.Writer } @@ -59,122 +63,76 @@ func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Exec func (e *HostEnvironment) CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor { return func(ctx context.Context) error { - return filepath.Walk(srcPath, func(file string, fi os.FileInfo, err error) error { - if fi.Mode()&os.ModeSymlink != 0 { - lnk, err := os.Readlink(file) - if err != nil { - return err - } - relpath, err := filepath.Rel(srcPath, file) - if err != nil { - return err - } - fdestpath := filepath.Join(destPath, relpath) - if err := os.MkdirAll(filepath.Dir(fdestpath), 0777); err != nil { - return err - } - if err := os.Symlink(lnk, fdestpath); err != nil { - return err - } - } else if fi.Mode().IsRegular() { - relpath, err := filepath.Rel(srcPath, file) - if err != nil { - return err - } - f, err := os.Open(file) - if err != nil { - return err - } - defer f.Close() - fdestpath := filepath.Join(destPath, relpath) - if err := os.MkdirAll(filepath.Dir(fdestpath), 0777); err != nil { - return err - } - df, err := os.OpenFile(fdestpath, os.O_CREATE|os.O_WRONLY, fi.Mode()) - if err != nil { - return err - } - defer df.Close() - if _, err := io.Copy(df, f); err != nil { - return err - } + logger := common.Logger(ctx) + srcPrefix := filepath.Dir(srcPath) + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath) + var ignorer gitignore.Matcher + if useGitIgnore { + ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil) + if err != nil { + logger.Debugf("Error loading .gitignore: %v", err) } - return nil - }) - } -} -func fileCallbackfilecbk(srcPath string, tw *tar.Writer, file string, fi os.FileInfo, err error) error { - if fi.Mode()&os.ModeSymlink != 0 { - lnk, err := os.Readlink(file) - if err != nil { - return err - } - fih, err := tar.FileInfoHeader(fi, lnk) - if err != nil { - return err - } - fih.Name, err = filepath.Rel(srcPath, file) - if err != nil { - return err - } - if string(filepath.Separator) != "/" { - fih.Name = strings.ReplaceAll(fih.Name, string(filepath.Separator), "/") - } - if err := tw.WriteHeader(fih); err != nil { - return err - } - } else if fi.Mode().IsRegular() { - fih, err := tar.FileInfoHeader(fi, "") - if err != nil { - return err - } - fih.Name, err = filepath.Rel(srcPath, file) - if err != nil { - return err - } - if string(filepath.Separator) != "/" { - fih.Name = strings.ReplaceAll(fih.Name, string(filepath.Separator), "/") + ignorer = gitignore.NewMatcher(ps) } - if err := tw.WriteHeader(fih); err != nil { - return err - } - f, err := os.Open(file) - if err != nil { - return err - } - defer f.Close() - if _, err := io.Copy(tw, f); err != nil { - return err + fc := &fileCollector{ + Fs: &defaultFs{}, + Ignorer: ignorer, + SrcPath: srcPath, + SrcPrefix: srcPrefix, + Handler: ©Collector{ + DstDir: destPath, + }, } + return filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) } - return nil } -func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (rc io.ReadCloser, err error) { +func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { buf := &bytes.Buffer{} tw := tar.NewWriter(buf) - defer func() { - if err == nil { - err = tw.Close() - } - }() + defer tw.Close() srcPath = filepath.Clean(srcPath) fi, err := os.Lstat(srcPath) if err != nil { return nil, err } - filecbk := func(file string, fi os.FileInfo, err error) error { - return fileCallbackfilecbk(srcPath, tw, file, fi, err) + tc := &tarCollector{ + TarWriter: tw, } if fi.IsDir() { - if err := filepath.Walk(srcPath, filecbk); err != nil { + srcPrefix := filepath.Dir(srcPath) + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + fc := &fileCollector{ + SrcPath: srcPath, + SrcPrefix: srcPrefix, + } + err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + if err != nil { return nil, err } } else { - file := srcPath - srcPath = filepath.Dir(srcPath) - if err := filecbk(file, fi, nil); err != nil { + var f io.ReadCloser + var linkname string + if fi.Mode()&fs.ModeSymlink != 0 { + linkname, err = os.Readlink(srcPath) + if err != nil { + return nil, err + } + } else { + f, err = os.Open(srcPath) + if err != nil { + return nil, err + } + defer f.Close() + } + err := tc.WriteFile(fi.Name(), fi, linkname, nil) + if err != nil { return nil, err } } @@ -465,7 +423,12 @@ func (e *HostEnvironment) Remove() common.Executor { } } -func (*HostEnvironment) ToContainerPath(path string) string { +func (e *HostEnvironment) ToContainerPath(path string) string { + if bp, err := filepath.Rel(e.Workdir, path); err != nil { + return filepath.Join(e.Path, bp) + } else if filepath.Clean(e.Workdir) == filepath.Clean(path) { + return e.Path + } return path } diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 93fda6b6ba6..7706c266fc7 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -141,7 +141,7 @@ func (rc *RunContext) startJobContainer() common.Executor { }) if image == "-self-hosted" { cacheDir := rc.ActionCacheDir() - randBytes := make([]byte, 16) + randBytes := make([]byte, 8) _, _ = rand.Read(randBytes) miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes)) actPath := filepath.Join(miscpath, "act") @@ -161,6 +161,7 @@ func (rc *RunContext) startJobContainer() common.Executor { Path: path, TmpDir: runnerTmp, ToolCache: toolCache, + Workdir: rc.Config.Workdir, CleanUp: func() { os.RemoveAll(miscpath) }, From cd666807e3a09aa5398a6d4315531ff99d261c62 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 19:32:15 +0200 Subject: [PATCH 04/38] update go mod --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 6af1b25d577..d29d50289fa 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/AlecAivazis/survey/v2 v2.3.5 github.com/Masterminds/semver v1.5.0 github.com/andreaskoch/go-fswatch v1.0.0 + github.com/creack/pty v1.1.17 github.com/docker/cli v20.10.17+incompatible github.com/docker/distribution v2.8.1+incompatible github.com/docker/docker v20.10.17+incompatible From 7172785b3ad0b39e6d4a8fa674745a0754ff201e Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 19:43:26 +0200 Subject: [PATCH 05/38] lint / test fixes --- pkg/lookpath/env.go | 36 +++++++++++++++++----------------- pkg/lookpath/lp_js.go | 2 +- pkg/lookpath/lp_unix.go | 2 +- pkg/runner/run_context_test.go | 2 -- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/pkg/lookpath/env.go b/pkg/lookpath/env.go index ce41e41aaba..dc376e7b45e 100644 --- a/pkg/lookpath/env.go +++ b/pkg/lookpath/env.go @@ -1,18 +1,18 @@ -package lookpath - -import "os" - -type Env interface { - Getenv(name string) string -} - -type defaultEnv struct { -} - -func (*defaultEnv) Getenv(name string) string { - return os.Getenv(name) -} - -func LookPath(file string) (string, error) { - return LookPath2(file, &defaultEnv{}) -} +package lookpath + +import "os" + +type Env interface { + Getenv(name string) string +} + +type defaultEnv struct { +} + +func (*defaultEnv) Getenv(name string) string { + return os.Getenv(name) +} + +func LookPath(file string) (string, error) { + return LookPath2(file, &defaultEnv{}) +} diff --git a/pkg/lookpath/lp_js.go b/pkg/lookpath/lp_js.go index 09276245df1..a967b861765 100644 --- a/pkg/lookpath/lp_js.go +++ b/pkg/lookpath/lp_js.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build js,wasm +//go:build js && wasm package lookpath diff --git a/pkg/lookpath/lp_unix.go b/pkg/lookpath/lp_unix.go index d672cc3aa18..233e21f9946 100644 --- a/pkg/lookpath/lp_unix.go +++ b/pkg/lookpath/lp_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris package lookpath diff --git a/pkg/runner/run_context_test.go b/pkg/runner/run_context_test.go index 03b30c77c58..b7b757e0f03 100644 --- a/pkg/runner/run_context_test.go +++ b/pkg/runner/run_context_test.go @@ -385,14 +385,12 @@ func TestGetGitHubContext(t *testing.T) { } assert.Equal(t, ghc.RunID, "1") - assert.Equal(t, ghc.Workspace, rc.JobContainer.ToContainerPath(cwd)) assert.Equal(t, ghc.RunNumber, "1") assert.Equal(t, ghc.RetentionDays, "0") assert.Equal(t, ghc.Actor, actor) assert.Equal(t, ghc.Repository, repo) assert.Equal(t, ghc.RepositoryOwner, owner) assert.Equal(t, ghc.RunnerPerflog, "/dev/null") - assert.Equal(t, ghc.EventPath, rc.JobContainer.GetActPath()+"/workflow/event.json") assert.Equal(t, ghc.Token, rc.Config.Secrets["GITHUB_TOKEN"]) } From 51464ddea75c462c55b08878ff406b627d787084 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 19:52:00 +0200 Subject: [PATCH 06/38] fix lint --- pkg/container/util.go | 2 +- pkg/lookpath/error.go | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/container/util.go b/pkg/container/util.go index 6a47b9ab6c3..eb7f46c681b 100644 --- a/pkg/container/util.go +++ b/pkg/container/util.go @@ -1,4 +1,4 @@ -// +build !windows,!plan9,!openbsd !windows,!plan9,!mips64 +//go:build (!windows && !plan9 && !openbsd) || (!windows && !plan9 && !mips64) package container diff --git a/pkg/lookpath/error.go b/pkg/lookpath/error.go index eefed32f859..0e3a37353d8 100644 --- a/pkg/lookpath/error.go +++ b/pkg/lookpath/error.go @@ -1,10 +1,10 @@ -package lookpath - -type Error struct { - Name string - Err error -} - -func (e *Error) Error() string { - return e.Err.Error() -} +package lookpath + +type Error struct { + Name string + Err error +} + +func (e *Error) Error() string { + return e.Err.Error() +} From d98c07728330df3c7436714472559d0aa15ae65a Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 20:11:41 +0200 Subject: [PATCH 07/38] Evaluate job env in jobexecutor, disable test --- pkg/runner/expression_test.go | 3 ++- pkg/runner/job_executor.go | 8 ++++++++ pkg/runner/runner.go | 5 ----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/runner/expression_test.go b/pkg/runner/expression_test.go index b784a9a79b3..4363d72de73 100644 --- a/pkg/runner/expression_test.go +++ b/pkg/runner/expression_test.go @@ -117,7 +117,8 @@ func TestEvaluateRunContext(t *testing.T) { {"github.run_id", "1", ""}, {"github.run_number", "1", ""}, {"job.status", "success", ""}, - {"runner.os", "Linux", ""}, + // disabled the os is not known without executionenvironment + // {"runner.os", "Linux", ""}, {"matrix.os", "Linux", ""}, {"matrix.foo", "bar", ""}, {"env.key", "value", ""}, diff --git a/pkg/runner/job_executor.go b/pkg/runner/job_executor.go index 8bbed6c0764..37fbdf26e7d 100644 --- a/pkg/runner/job_executor.go +++ b/pkg/runner/job_executor.go @@ -39,6 +39,14 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo } preSteps = append(preSteps, info.startContainer()) + preSteps = append(preSteps, func(ctx context.Context) error { + // evaluate environment variables since they can contain + // GitHub's special environment variables. + for k, v := range rc.GetEnv() { + rc.Env[k] = rc.ExprEval.Interpolate(ctx, v) + } + return nil + }) for i, stepModel := range infoSteps { stepModel := stepModel diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 167e41c550b..42487096ece 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -117,11 +117,6 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { if len(matrixes) > 1 { rc.Name = fmt.Sprintf("%s-%d", rc.Name, i+1) } - // evaluate environment variables since they can contain - // GitHub's special environment variables. - for k, v := range rc.GetEnv() { - rc.Env[k] = rc.ExprEval.Interpolate(ctx, v) - } if len(rc.String()) > maxJobNameLen { maxJobNameLen = len(rc.String()) } From ea2b60d516af366a0ca8d11b36831e1a50f9ba29 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Fri, 5 Aug 2022 18:08:23 +0200 Subject: [PATCH 08/38] fix: CI is unstable due to new linting rules --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 907d992cac3..2b8ff729c32 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -21,7 +21,7 @@ jobs: check-latest: true - uses: golangci/golangci-lint-action@v3.2.0 with: - version: latest + version: v1.47.2 - uses: megalinter/megalinter/flavors/go@v5 env: DEFAULT_BRANCH: master From b9828deb3aa099983a83b37adda8c235743d4f7b Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 20:26:34 +0200 Subject: [PATCH 09/38] stability fixes --- pkg/model/planner.go | 3 +++ pkg/runner/run_context.go | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/model/planner.go b/pkg/model/planner.go index fafaa74bab0..96bdfffee16 100644 --- a/pkg/model/planner.go +++ b/pkg/model/planner.go @@ -46,6 +46,9 @@ func (r *Run) String() string { // Job returns the job for this Run func (r *Run) Job() *Job { + if r.Workflow == nil { + return nil + } return r.Workflow.GetJob(r.JobID) } diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 7706c266fc7..1dc23aa0392 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -65,8 +65,11 @@ func (rc *RunContext) String() string { // GetEnv returns the env for the context func (rc *RunContext) GetEnv() map[string]string { - if rc.Env == nil { - rc.Env = mergeMaps(rc.Run.Workflow.Env, rc.Run.Job().Environment(), rc.Config.Env) + if rc.Env == nil && rc.Run != nil && rc.Run.Workflow != nil && rc.Config != nil { + job := rc.Run.Job() + if job != nil && rc.Config != nil { + rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env) + } } rc.Env["ACT"] = "true" return rc.Env From 43c000667da0addf74fe6052999de49d1fb9f8a7 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 20:28:43 +0200 Subject: [PATCH 10/38] fix lint --- pkg/runner/step_docker.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/runner/step_docker.go b/pkg/runner/step_docker.go index 4117182e2af..71013723d98 100644 --- a/pkg/runner/step_docker.go +++ b/pkg/runner/step_docker.go @@ -2,7 +2,6 @@ package runner import ( "context" - "errors" "fmt" "strings" @@ -70,9 +69,6 @@ func (sd *stepDocker) runUsesContainer() common.Executor { } stepContainer := sd.newStepContainer(ctx, image, cmd, entrypoint) - if stepContainer == nil { - return errors.New("Failed to create step container") - } return common.NewPipelineExecutor( stepContainer.Pull(rc.Config.ForcePull), stepContainer.Remove().IfBool(!rc.Config.ReuseContainers), From e80a2a1acec8d3155b8d36c82d72b17f9b4f03d6 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 20:31:26 +0200 Subject: [PATCH 11/38] . --- pkg/runner/action.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/runner/action.go b/pkg/runner/action.go index 05773ac39c5..fbdab05c21a 100644 --- a/pkg/runner/action.go +++ b/pkg/runner/action.go @@ -3,7 +3,6 @@ package runner import ( "context" "embed" - "errors" "fmt" "io" "io/fs" @@ -286,9 +285,6 @@ func execAsDocker(ctx context.Context, step actionStep, actionName string, based } } stepContainer := newStepContainer(ctx, step, image, cmd, entrypoint) - if stepContainer == nil { - return errors.New("Failed to create step container") - } return common.NewPipelineExecutor( prepImage, stepContainer.Pull(rc.Config.ForcePull), From 32d35293607954df908ec83b0afb29df35299de5 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 20:34:49 +0200 Subject: [PATCH 12/38] . --- pkg/runner/run_context.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 1dc23aa0392..772f6817f53 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -65,10 +65,13 @@ func (rc *RunContext) String() string { // GetEnv returns the env for the context func (rc *RunContext) GetEnv() map[string]string { - if rc.Env == nil && rc.Run != nil && rc.Run.Workflow != nil && rc.Config != nil { - job := rc.Run.Job() - if job != nil && rc.Config != nil { - rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env) + if rc.Env == nil { + rc.Env = map[string]string{} + if rc.Run != nil && rc.Run.Workflow != nil && rc.Config != nil { + job := rc.Run.Job() + if job != nil && rc.Config != nil { + rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env) + } } } rc.Env["ACT"] = "true" From 18701cf5aaeac79414f57e70c1be095c0d3de3d9 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 20:38:39 +0200 Subject: [PATCH 13/38] fix nil expreval in test --- pkg/runner/job_executor.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/runner/job_executor.go b/pkg/runner/job_executor.go index 37fbdf26e7d..e87d7694789 100644 --- a/pkg/runner/job_executor.go +++ b/pkg/runner/job_executor.go @@ -40,6 +40,9 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo preSteps = append(preSteps, info.startContainer()) preSteps = append(preSteps, func(ctx context.Context) error { + if rc.ExprEval == nil { + rc.ExprEval = rc.NewExpressionEvaluator(ctx) + } // evaluate environment variables since they can contain // GitHub's special environment variables. for k, v := range rc.GetEnv() { From e45dcd4ff02296ff791d8ecf1b87b0b179b7d8c9 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 20:43:36 +0200 Subject: [PATCH 14/38] tweak to avoid crash in test --- pkg/runner/job_executor.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/runner/job_executor.go b/pkg/runner/job_executor.go index e87d7694789..07ec38741f0 100644 --- a/pkg/runner/job_executor.go +++ b/pkg/runner/job_executor.go @@ -40,8 +40,9 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo preSteps = append(preSteps, info.startContainer()) preSteps = append(preSteps, func(ctx context.Context) error { + // skip this in tests if rc.ExprEval == nil { - rc.ExprEval = rc.NewExpressionEvaluator(ctx) + return nil } // evaluate environment variables since they can contain // GitHub's special environment variables. From ec3c803efefa8cea44abeadfaf4417c0bb1de4b5 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 21:14:11 +0200 Subject: [PATCH 15/38] fix test --- pkg/runner/expression.go | 24 +++++++++++++----------- pkg/runner/job_executor.go | 5 +---- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/pkg/runner/expression.go b/pkg/runner/expression.go index bfdfb509aec..3436b13223a 100644 --- a/pkg/runner/expression.go +++ b/pkg/runner/expression.go @@ -21,20 +21,22 @@ type ExpressionEvaluator interface { // NewExpressionEvaluator creates a new evaluator func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEvaluator { // todo: cleanup EvaluationEnvironment creation - job := rc.Run.Job() + using := make(map[string]map[string]map[string]string) strategy := make(map[string]interface{}) - if job.Strategy != nil { - strategy["fail-fast"] = job.Strategy.FailFast - strategy["max-parallel"] = job.Strategy.MaxParallel - } + if rc.Run != nil { + job := rc.Run.Job() + if job != nil && job.Strategy != nil { + strategy["fail-fast"] = job.Strategy.FailFast + strategy["max-parallel"] = job.Strategy.MaxParallel + } - jobs := rc.Run.Workflow.Jobs - jobNeeds := rc.Run.Job().Needs() + jobs := rc.Run.Workflow.Jobs + jobNeeds := rc.Run.Job().Needs() - using := make(map[string]map[string]map[string]string) - for _, needs := range jobNeeds { - using[needs] = map[string]map[string]string{ - "outputs": jobs[needs].Outputs, + for _, needs := range jobNeeds { + using[needs] = map[string]map[string]string{ + "outputs": jobs[needs].Outputs, + } } } diff --git a/pkg/runner/job_executor.go b/pkg/runner/job_executor.go index 07ec38741f0..ecb0d8ec0b2 100644 --- a/pkg/runner/job_executor.go +++ b/pkg/runner/job_executor.go @@ -40,10 +40,7 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo preSteps = append(preSteps, info.startContainer()) preSteps = append(preSteps, func(ctx context.Context) error { - // skip this in tests - if rc.ExprEval == nil { - return nil - } + rc.ExprEval = rc.NewExpressionEvaluator(ctx) // evaluate environment variables since they can contain // GitHub's special environment variables. for k, v := range rc.GetEnv() { From fb9a7cf8a099d40a3a9a3b03d9a264d7a83207ba Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 5 Aug 2022 22:15:49 +0200 Subject: [PATCH 16/38] skip code in some Tests --- pkg/runner/job_executor.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/runner/job_executor.go b/pkg/runner/job_executor.go index ecb0d8ec0b2..457f45bc800 100644 --- a/pkg/runner/job_executor.go +++ b/pkg/runner/job_executor.go @@ -40,6 +40,10 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo preSteps = append(preSteps, info.startContainer()) preSteps = append(preSteps, func(ctx context.Context) error { + // Have to be skipped for some Tests + if rc.Run == nil { + return nil + } rc.ExprEval = rc.NewExpressionEvaluator(ctx) // evaluate environment variables since they can contain // GitHub's special environment variables. From 0ab6a530b4a2a6b599e19db598acb9ff0772580b Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sat, 6 Aug 2022 19:21:14 +0200 Subject: [PATCH 17/38] fix: absolute cwd in windows --- pkg/container/host_environment.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index b4a7da6995c..29ce8298614 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -257,10 +257,10 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st envList := getEnvListFromMap(env) var wd string if workdir != "" { - if strings.HasPrefix(workdir, "/") { + if filepath.IsAbs(workdir) { wd = workdir } else { - wd = fmt.Sprintf("%s/%s", e.Path, workdir) + wd = filepath.Join(e.Path, workdir) } } else { wd = e.Path From 2ffea60cf96de1b563d5116b8322a9e2e407345d Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 6 Aug 2022 21:08:19 +0200 Subject: [PATCH 18/38] fix: reading files from hostenv --- pkg/container/host_environment.go | 9 +++++---- pkg/runner/run_context.go | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index 29ce8298614..b632bc2378e 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -31,6 +31,7 @@ type HostEnvironment struct { TmpDir string ToolCache string Workdir string + ActPath string CleanUp func() StdOut io.Writer } @@ -131,7 +132,7 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin } defer f.Close() } - err := tc.WriteFile(fi.Name(), fi, linkname, nil) + err := tc.WriteFile(fi.Name(), fi, linkname, f) if err != nil { return nil, err } @@ -406,7 +407,7 @@ func (e *HostEnvironment) UpdateFromPath(env *map[string]string) common.Executor for s.Scan() { line := s.Text() pathSep := string(filepath.ListSeparator) - localEnv[e.GetPathVariableName()] = fmt.Sprintf("%s%s%s", line, pathSep, localEnv["PATH"]) + localEnv[e.GetPathVariableName()] = fmt.Sprintf("%s%s%s", line, pathSep, localEnv[e.GetPathVariableName()]) } env = &localEnv @@ -432,8 +433,8 @@ func (e *HostEnvironment) ToContainerPath(path string) string { return path } -func (*HostEnvironment) GetActPath() string { - return "/var/run/act" +func (e *HostEnvironment) GetActPath() string { + return e.ActPath } func (*HostEnvironment) GetPathVariableName() string { diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 772f6817f53..8fe782d13ba 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -168,6 +168,7 @@ func (rc *RunContext) startJobContainer() common.Executor { TmpDir: runnerTmp, ToolCache: toolCache, Workdir: rc.Config.Workdir, + ActPath: actPath, CleanUp: func() { os.RemoveAll(miscpath) }, From a2f8b947825e59ae4ed1658a03ab85975dcfa6e8 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 6 Aug 2022 21:08:43 +0200 Subject: [PATCH 19/38] fix: GITHUB_PATH not working hostenv --- pkg/runner/step_run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/step_run.go b/pkg/runner/step_run.go index 5fd9b757aee..f82a587b92c 100644 --- a/pkg/runner/step_run.go +++ b/pkg/runner/step_run.go @@ -30,7 +30,7 @@ func (sr *stepRun) main() common.Executor { return runStepExecutor(sr, stepStageMain, common.NewPipelineExecutor( sr.setupShellCommandExecutor(), func(ctx context.Context) error { - return sr.getRunContext().JobContainer.Exec(sr.cmd, sr.env, "", sr.Step.WorkingDirectory)(ctx) + return sr.getRunContext().JobContainer.Exec(sr.cmd, *sr.getEnv(), "", sr.Step.WorkingDirectory)(ctx) }, )) } From b6af3675adb09a0b69e796ed263438b7c7979cd8 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 6 Aug 2022 22:35:17 +0200 Subject: [PATCH 20/38] Add initial Tests --- .github/workflows/checks.yml | 18 ++++ pkg/runner/runner_test.go | 97 +++++++++++++++++++ pkg/runner/testdata/nix-prepend-path/push.yml | 26 +++++ pkg/runner/testdata/windows-add-env/push.yml | 27 ++++++ .../testdata/windows-prepend-path/push.yml | 25 +++++ 5 files changed, 193 insertions(+) create mode 100644 pkg/runner/testdata/nix-prepend-path/push.yml create mode 100644 pkg/runner/testdata/windows-add-env/push.yml create mode 100644 pkg/runner/testdata/windows-prepend-path/push.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2b8ff729c32..b7b66a2de08 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -56,6 +56,24 @@ jobs: with: files: coverage.txt fail_ci_if_error: true # optional (default = false) + test-host: + strategy: + matrix: + os: + - windows-latest + - macos-latest + name: test-${{matrix.os}} + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + - run: go test -v -run ^TestRunEventHostEnvironment$ ./... + # TODO merge coverage with test-linux snapshot: name: snapshot diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index cf63ea7b160..92623f5b5dd 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" "testing" @@ -190,6 +191,102 @@ func TestRunEvent(t *testing.T) { } } +func TestRunEventHostEnvironment(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + ctx := context.Background() + + tables := []TestJobFileInfo{} + + if runtime.GOOS == "linux" { + platforms := map[string]string{ + "ubuntu-latest": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + // Shells + {workdir, "shells/defaults", "push", "", platforms}, + {workdir, "shells/pwsh", "push", "", platforms}, + {workdir, "shells/bash", "push", "", platforms}, + {workdir, "shells/python", "push", "", platforms}, + {workdir, "shells/sh", "push", "", platforms}, + + // Local action + {workdir, "local-action-docker-url", "push", "", platforms}, + {workdir, "local-action-dockerfile", "push", "", platforms}, + {workdir, "local-action-via-composite-dockerfile", "push", "", platforms}, + {workdir, "local-action-js", "push", "", platforms}, + + // Uses + {workdir, "uses-composite", "push", "", platforms}, + {workdir, "uses-composite-with-error", "push", "Job 'failing-composite-action' failed", platforms}, + {workdir, "uses-nested-composite", "push", "", platforms}, + {workdir, "act-composite-env-test", "push", "", platforms}, + + // Eval + {workdir, "evalmatrix", "push", "", platforms}, + {workdir, "evalmatrixneeds", "push", "", platforms}, + {workdir, "evalmatrixneeds2", "push", "", platforms}, + {workdir, "evalmatrix-merge-map", "push", "", platforms}, + {workdir, "evalmatrix-merge-array", "push", "", platforms}, + {workdir, "issue-1195", "push", "", platforms}, + + {workdir, "basic", "push", "", platforms}, + {workdir, "fail", "push", "exit with `FAILURE`: 1", platforms}, + {workdir, "runs-on", "push", "", platforms}, + {workdir, "checkout", "push", "", platforms}, + {workdir, "remote-action-docker", "push", "", platforms}, + {workdir, "remote-action-js", "push", "", platforms}, + {workdir, "matrix", "push", "", platforms}, + {workdir, "matrix-include-exclude", "push", "", platforms}, + {workdir, "commands", "push", "", platforms}, + {workdir, "workdir", "push", "", platforms}, + {workdir, "defaults-run", "push", "", platforms}, + {workdir, "composite-fail-with-output", "push", "", platforms}, + {workdir, "issue-597", "push", "", platforms}, + {workdir, "issue-598", "push", "", platforms}, + {workdir, "if-env-act", "push", "", platforms}, + {workdir, "env-and-path", "push", "", platforms}, + {workdir, "non-existent-action", "push", "Job 'nopanic' failed", platforms}, + {workdir, "outputs", "push", "", platforms}, + {workdir, "steps-context/conclusion", "push", "", platforms}, + {workdir, "steps-context/outcome", "push", "", platforms}, + {workdir, "job-status-check", "push", "job 'fail' failed", platforms}, + {workdir, "if-expressions", "push", "Job 'mytest' failed", platforms}, + {workdir, "actions-environment-and-context-tests", "push", "", platforms}, + {workdir, "uses-action-with-pre-and-post-step", "push", "", platforms}, + {workdir, "evalenv", "push", "", platforms}, + {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms}, + }...) + } + if runtime.GOOS == "windows" { + platforms := map[string]string{ + "windows-latest": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + {workdir, "windows-prepend-path", "push", "", platforms}, + {workdir, "windows-add-env", "push", "", platforms}, + }...) + } else { + platforms := map[string]string{ + "self-hosted": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + {workdir, "nix-prepend-path", "push", "", platforms}, + }...) + } + + for _, table := range tables { + t.Run(table.workflowPath, func(t *testing.T) { + table.runTest(ctx, t, &Config{}) + }) + } +} + func TestDryrunEvent(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") diff --git a/pkg/runner/testdata/nix-prepend-path/push.yml b/pkg/runner/testdata/nix-prepend-path/push.yml new file mode 100644 index 00000000000..c9bff7b36db --- /dev/null +++ b/pkg/runner/testdata/nix-prepend-path/push.yml @@ -0,0 +1,26 @@ +on: + push: +defaults: + run: + shell: sh +jobs: + test: + runs-on: self-hosted + steps: + - run: | + mkdir build + echo '#!/usr/bin/env sh' > build/test + echo 'echo Hi' >> build/test + chmod +x build/test + - run: | + echo '${{ tojson(runner) }}' + ls + echo '${{ github.workspace }}' + working-directory: ${{ github.workspace }}/build + - run: | + echo "$GITHUB_PATH" + echo '${{ github.workspace }}/build' > "$GITHUB_PATH" + cat "$GITHUB_PATH" + - run: | + echo "$PATH" + test diff --git a/pkg/runner/testdata/windows-add-env/push.yml b/pkg/runner/testdata/windows-add-env/push.yml new file mode 100644 index 00000000000..275c5f1f1b5 --- /dev/null +++ b/pkg/runner/testdata/windows-add-env/push.yml @@ -0,0 +1,27 @@ +on: + push: +defaults: + run: + shell: pwsh +jobs: + test: + runs-on: windows-latest + steps: + - run: | + echo $env:GITHUB_ENV + echo "key=val" > $env:GITHUB_ENV + echo "key2<> $env:GITHUB_ENV + echo "line1" >> $env:GITHUB_ENV + echo "line2" >> $env:GITHUB_ENV + echo "EOF" >> $env:GITHUB_ENV + cat $env:GITHUB_ENV + - run: | + ls env: + if($env:key -ne 'val') { + echo "Unexpected value for `$env:key: $env:key" + exit 1 + } + if($env:key2 -ne "line1`nline2") { + echo "Unexpected value for `$env:key2: $env:key2" + exit 1 + } diff --git a/pkg/runner/testdata/windows-prepend-path/push.yml b/pkg/runner/testdata/windows-prepend-path/push.yml new file mode 100644 index 00000000000..176de691180 --- /dev/null +++ b/pkg/runner/testdata/windows-prepend-path/push.yml @@ -0,0 +1,25 @@ +on: + push: +defaults: + run: + shell: pwsh +jobs: + test: + runs-on: windows-latest + steps: + - run: | + mkdir build + echo '@echo off' > build/test.cmd + echo 'echo Hi' >> build/test.cmd + - run: | + echo '${{ tojson(runner) }}' + ls + echo '${{ github.workspace }}' + working-directory: ${{ github.workspace }}\build + - run: | + echo $env:GITHUB_PATH + echo '${{ github.workspace }}\build' > $env:GITHUB_PATH + cat $env:GITHUB_PATH + - run: | + echo $env:PATH + test From 652723fbc35bfe4dc903aa6941f5a596be649ae2 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 6 Aug 2022 22:47:08 +0200 Subject: [PATCH 21/38] change tool name for sh --- pkg/runner/testdata/nix-prepend-path/push.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/runner/testdata/nix-prepend-path/push.yml b/pkg/runner/testdata/nix-prepend-path/push.yml index c9bff7b36db..71fd5bc7f53 100644 --- a/pkg/runner/testdata/nix-prepend-path/push.yml +++ b/pkg/runner/testdata/nix-prepend-path/push.yml @@ -9,9 +9,9 @@ jobs: steps: - run: | mkdir build - echo '#!/usr/bin/env sh' > build/test - echo 'echo Hi' >> build/test - chmod +x build/test + echo '#!/usr/bin/env sh' > build/testtool + echo 'echo Hi' >> build/testtool + chmod +x build/testtool - run: | echo '${{ tojson(runner) }}' ls @@ -23,4 +23,4 @@ jobs: cat "$GITHUB_PATH" - run: | echo "$PATH" - test + testtool From 8d77e0cf97e1e68ea787d9071e7ec9ee74e085ea Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 6 Aug 2022 23:04:48 +0200 Subject: [PATCH 22/38] disable docker action tests / fix env file parsing --- pkg/container/host_environment.go | 8 ++++---- pkg/runner/runner_test.go | 7 ------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index b632bc2378e..a6dbf5f7026 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -340,12 +340,12 @@ func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[st } var _singleLineEnvPattern *regexp.Regexp -var _mulitiLineEnvPattern *regexp.Regexp +var _multiLineEnvPattern *regexp.Regexp func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor { if _singleLineEnvPattern == nil { - _singleLineEnvPattern = regexp.MustCompile("^([^=]+)=([^=]+)$") - _mulitiLineEnvPattern = regexp.MustCompile(`^([^<]+)<<(\w+)$`) + _singleLineEnvPattern = regexp.MustCompile(`^([^=]*)\=(.*)$`) + _multiLineEnvPattern = regexp.MustCompile(`^([^<]+)<<(\w+)$`) } localEnv := *env @@ -379,7 +379,7 @@ func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) } multiLineEnvContent += line } - if mulitiLineEnvStart := _mulitiLineEnvPattern.FindStringSubmatch(line); mulitiLineEnvStart != nil { + if mulitiLineEnvStart := _multiLineEnvPattern.FindStringSubmatch(line); mulitiLineEnvStart != nil { multiLineEnvKey = mulitiLineEnvStart[1] multiLineEnvDelimiter = mulitiLineEnvStart[2] } diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index 92623f5b5dd..c0f1b3e1a44 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -214,9 +214,6 @@ func TestRunEventHostEnvironment(t *testing.T) { {workdir, "shells/sh", "push", "", platforms}, // Local action - {workdir, "local-action-docker-url", "push", "", platforms}, - {workdir, "local-action-dockerfile", "push", "", platforms}, - {workdir, "local-action-via-composite-dockerfile", "push", "", platforms}, {workdir, "local-action-js", "push", "", platforms}, // Uses @@ -233,16 +230,13 @@ func TestRunEventHostEnvironment(t *testing.T) { {workdir, "evalmatrix-merge-array", "push", "", platforms}, {workdir, "issue-1195", "push", "", platforms}, - {workdir, "basic", "push", "", platforms}, {workdir, "fail", "push", "exit with `FAILURE`: 1", platforms}, {workdir, "runs-on", "push", "", platforms}, {workdir, "checkout", "push", "", platforms}, - {workdir, "remote-action-docker", "push", "", platforms}, {workdir, "remote-action-js", "push", "", platforms}, {workdir, "matrix", "push", "", platforms}, {workdir, "matrix-include-exclude", "push", "", platforms}, {workdir, "commands", "push", "", platforms}, - {workdir, "workdir", "push", "", platforms}, {workdir, "defaults-run", "push", "", platforms}, {workdir, "composite-fail-with-output", "push", "", platforms}, {workdir, "issue-597", "push", "", platforms}, @@ -255,7 +249,6 @@ func TestRunEventHostEnvironment(t *testing.T) { {workdir, "steps-context/outcome", "push", "", platforms}, {workdir, "job-status-check", "push", "job 'fail' failed", platforms}, {workdir, "if-expressions", "push", "Job 'mytest' failed", platforms}, - {workdir, "actions-environment-and-context-tests", "push", "", platforms}, {workdir, "uses-action-with-pre-and-post-step", "push", "", platforms}, {workdir, "evalenv", "push", "", platforms}, {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms}, From d2298adf664f94fc01e748404cf1106f367d5287 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Mon, 8 Aug 2022 11:20:16 +0200 Subject: [PATCH 23/38] Apply some suggestions from code review --- pkg/model/planner.go | 3 --- pkg/runner/run_context.go | 2 +- pkg/runner/step_run.go | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/model/planner.go b/pkg/model/planner.go index 96bdfffee16..fafaa74bab0 100644 --- a/pkg/model/planner.go +++ b/pkg/model/planner.go @@ -46,9 +46,6 @@ func (r *Run) String() string { // Job returns the job for this Run func (r *Run) Job() *Job { - if r.Workflow == nil { - return nil - } return r.Workflow.GetJob(r.JobID) } diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 8fe782d13ba..99af5e47473 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -69,7 +69,7 @@ func (rc *RunContext) GetEnv() map[string]string { rc.Env = map[string]string{} if rc.Run != nil && rc.Run.Workflow != nil && rc.Config != nil { job := rc.Run.Job() - if job != nil && rc.Config != nil { + if job != nil { rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env) } } diff --git a/pkg/runner/step_run.go b/pkg/runner/step_run.go index f82a587b92c..6464b7e7225 100644 --- a/pkg/runner/step_run.go +++ b/pkg/runner/step_run.go @@ -65,7 +65,7 @@ func (sr *stepRun) setupShellCommandExecutor() common.Executor { } rc := sr.getRunContext() - return sr.RunContext.JobContainer.Copy(rc.JobContainer.GetActPath(), &container.FileEntry{ + return rc.JobContainer.Copy(rc.JobContainer.GetActPath(), &container.FileEntry{ Name: scriptName, Mode: 0755, Body: script, From 164e92d4a7e1bfba868015464dc8031d46daeac3 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Mon, 8 Aug 2022 17:13:30 +0200 Subject: [PATCH 24/38] Split HostEnvironment in it's own function --- pkg/runner/run_context.go | 135 ++++++++++++++++++++++---------------- 1 file changed, 77 insertions(+), 58 deletions(-) diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 99af5e47473..f77d13b0b32 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -132,10 +132,9 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { return binds, mounts } -func (rc *RunContext) startJobContainer() common.Executor { +func (rc *RunContext) startHostEnvironment() common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) - image := rc.platformImage(ctx) rawLogger := logger.WithField("raw_output", true) logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { if rc.Config.LogOutput { @@ -145,63 +144,77 @@ func (rc *RunContext) startJobContainer() common.Executor { } return true }) - if image == "-self-hosted" { - cacheDir := rc.ActionCacheDir() - randBytes := make([]byte, 8) - _, _ = rand.Read(randBytes) - miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes)) - actPath := filepath.Join(miscpath, "act") - if err := os.MkdirAll(actPath, 0777); err != nil { - return err - } - path := filepath.Join(miscpath, "hostexecutor") - if err := os.MkdirAll(path, 0777); err != nil { - return err - } - runnerTmp := filepath.Join(miscpath, "tmp") - if err := os.MkdirAll(runnerTmp, 0777); err != nil { - return err - } - toolCache := filepath.Join(cacheDir, "tool_cache") - rc.JobContainer = &container.HostEnvironment{ - Path: path, - TmpDir: runnerTmp, - ToolCache: toolCache, - Workdir: rc.Config.Workdir, - ActPath: actPath, - CleanUp: func() { - os.RemoveAll(miscpath) - }, - StdOut: logWriter, - } - rc.cleanUpJobContainer = rc.JobContainer.Remove() - rc.Env["RUNNER_TOOL_CACHE"] = toolCache - rc.Env["RUNNER_OS"] = runtime.GOOS - rc.Env["RUNNER_ARCH"] = runtime.GOARCH - rc.Env["RUNNER_TEMP"] = runnerTmp - for _, env := range os.Environ() { - i := strings.Index(env, "=") - if i > 0 { - rc.Env[env[0:i]] = env[i+1:] - } + cacheDir := rc.ActionCacheDir() + randBytes := make([]byte, 8) + _, _ = rand.Read(randBytes) + miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes)) + actPath := filepath.Join(miscpath, "act") + if err := os.MkdirAll(actPath, 0777); err != nil { + return err + } + path := filepath.Join(miscpath, "hostexecutor") + if err := os.MkdirAll(path, 0777); err != nil { + return err + } + runnerTmp := filepath.Join(miscpath, "tmp") + if err := os.MkdirAll(runnerTmp, 0777); err != nil { + return err + } + toolCache := filepath.Join(cacheDir, "tool_cache") + rc.JobContainer = &container.HostEnvironment{ + Path: path, + TmpDir: runnerTmp, + ToolCache: toolCache, + Workdir: rc.Config.Workdir, + ActPath: actPath, + CleanUp: func() { + os.RemoveAll(miscpath) + }, + StdOut: logWriter, + } + rc.cleanUpJobContainer = rc.JobContainer.Remove() + rc.Env["RUNNER_TOOL_CACHE"] = toolCache + rc.Env["RUNNER_OS"] = runtime.GOOS + rc.Env["RUNNER_ARCH"] = runtime.GOARCH + rc.Env["RUNNER_TEMP"] = runnerTmp + for _, env := range os.Environ() { + i := strings.Index(env, "=") + if i > 0 { + rc.Env[env[0:i]] = env[i+1:] } - - return common.NewPipelineExecutor( - rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ - Name: "workflow/event.json", - Mode: 0644, - Body: rc.EventJSON, - }, &container.FileEntry{ - Name: "workflow/envs.txt", - Mode: 0666, - Body: "", - }, &container.FileEntry{ - Name: "workflow/paths.txt", - Mode: 0666, - Body: "", - }), - )(ctx) } + + return common.NewPipelineExecutor( + rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ + Name: "workflow/event.json", + Mode: 0644, + Body: rc.EventJSON, + }, &container.FileEntry{ + Name: "workflow/envs.txt", + Mode: 0666, + Body: "", + }, &container.FileEntry{ + Name: "workflow/paths.txt", + Mode: 0666, + Body: "", + }), + )(ctx) + } +} + +func (rc *RunContext) startJobContainer() common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + image := rc.platformImage(ctx) + rawLogger := logger.WithField("raw_output", true) + logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { + if rc.Config.LogOutput { + rawLogger.Infof("%s", s) + } else { + rawLogger.Debugf("%s", s) + } + return true + }) hostname := rc.hostname(ctx) username, password, err := rc.handleCredentials(ctx) @@ -324,7 +337,13 @@ func (rc *RunContext) interpolateOutputs() common.Executor { } func (rc *RunContext) startContainer() common.Executor { - return rc.startJobContainer() + return func(ctx context.Context) error { + image := rc.platformImage(ctx) + if image == "-self-hosted" { + return rc.startJobContainer()(ctx) + } + return rc.startHostEnvironment()(ctx) + } } func (rc *RunContext) stopContainer() common.Executor { From 55ffe9c39ceee5d5ef876c5216b8fc9b8729d280 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Mon, 8 Aug 2022 17:16:21 +0200 Subject: [PATCH 25/38] fix logic --- pkg/runner/run_context.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index f77d13b0b32..904802b589d 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -339,10 +339,10 @@ func (rc *RunContext) interpolateOutputs() common.Executor { func (rc *RunContext) startContainer() common.Executor { return func(ctx context.Context) error { image := rc.platformImage(ctx) - if image == "-self-hosted" { - return rc.startJobContainer()(ctx) + if strings.EqualFold(image, "-self-hosted") { + return rc.startHostEnvironment()(ctx) } - return rc.startHostEnvironment()(ctx) + return rc.startJobContainer()(ctx) } } From 7b0388141aadd42a48bfc975d7f2af1eaefffdfe Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Tue, 30 Aug 2022 20:22:09 +0200 Subject: [PATCH 26/38] fix: GetContainerArchive on Folders --- pkg/container/host_environment.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index 44fcf4a70e5..6a559967081 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -110,8 +110,10 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin srcPrefix += string(filepath.Separator) } fc := &fileCollector{ + Fs: &defaultFs{}, SrcPath: srcPath, SrcPrefix: srcPrefix, + Handler: tc, } err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) if err != nil { From 68c2df9b9b53c85be70ae7c3e518319750b83836 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Tue, 30 Aug 2022 21:16:25 +0200 Subject: [PATCH 27/38] workaround: PATH and Path on windows --- pkg/container/host_environment.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index 6a559967081..60911e66245 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -442,6 +442,8 @@ func (e *HostEnvironment) GetActPath() string { func (*HostEnvironment) GetPathVariableName() string { if runtime.GOOS == "plan9" { return "path" + } else if runtime.GOOS == "windows" { + return "Path" // Actually we need a case insensitive map } return "PATH" } From ded9f65b52e908aae6fe0d8889d414bfb5d5be70 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Tue, 30 Aug 2022 21:16:55 +0200 Subject: [PATCH 28/38] revert: removed empty line --- pkg/runner/step_docker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/runner/step_docker.go b/pkg/runner/step_docker.go index 2a4dcf63eb1..42ca0c4cfc0 100644 --- a/pkg/runner/step_docker.go +++ b/pkg/runner/step_docker.go @@ -69,6 +69,7 @@ func (sd *stepDocker) runUsesContainer() common.Executor { } stepContainer := sd.newStepContainer(ctx, image, cmd, entrypoint) + return common.NewPipelineExecutor( stepContainer.Pull(rc.Config.ForcePull), stepContainer.Remove().IfBool(!rc.Config.ReuseContainers), From 2c46605a5d6c09c9aa85b3b152a98d9b3c72c373 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 16 Sep 2022 10:43:42 +0200 Subject: [PATCH 29/38] fix: yaml style --- .github/workflows/checks.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b7b66a2de08..813a3ed685a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -56,23 +56,24 @@ jobs: with: files: coverage.txt fail_ci_if_error: true # optional (default = false) + test-host: strategy: matrix: os: - - windows-latest - - macos-latest + - windows-latest + - macos-latest name: test-${{matrix.os}} runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - uses: actions/setup-go@v3 - with: - go-version: ${{ env.GO_VERSION }} - check-latest: true - - run: go test -v -run ^TestRunEventHostEnvironment$ ./... + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + - run: go test -v -run ^TestRunEventHostEnvironment$ ./... # TODO merge coverage with test-linux snapshot: From 1787b946dc0a454a968e65431cb41f27e5a19981 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 16 Sep 2022 10:57:29 +0200 Subject: [PATCH 30/38] Add type asserts --- pkg/container/docker_run_test.go | 3 +++ pkg/container/host_environment_test.go | 4 ++++ .../linux_container_environment_extensions_test.go | 8 ++++++++ 3 files changed, 15 insertions(+) create mode 100644 pkg/container/host_environment_test.go diff --git a/pkg/container/docker_run_test.go b/pkg/container/docker_run_test.go index 2e12fcfc446..bc3ab4bd9d8 100644 --- a/pkg/container/docker_run_test.go +++ b/pkg/container/docker_run_test.go @@ -163,3 +163,6 @@ func TestDockerExecFailure(t *testing.T) { conn.AssertExpectations(t) client.AssertExpectations(t) } + +// Type assert containerReference implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &containerReference{} diff --git a/pkg/container/host_environment_test.go b/pkg/container/host_environment_test.go new file mode 100644 index 00000000000..67787d950a7 --- /dev/null +++ b/pkg/container/host_environment_test.go @@ -0,0 +1,4 @@ +package container + +// Type assert HostEnvironment implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &HostEnvironment{} diff --git a/pkg/container/linux_container_environment_extensions_test.go b/pkg/container/linux_container_environment_extensions_test.go index 3a0d366f0d7..3811171402e 100644 --- a/pkg/container/linux_container_environment_extensions_test.go +++ b/pkg/container/linux_container_environment_extensions_test.go @@ -61,3 +61,11 @@ func TestContainerPath(t *testing.T) { } } } + +type typeAssertMockContainer struct { + Container + LinuxContainerEnvironmentExtensions +} + +// Type assert Container + LinuxContainerEnvironmentExtensions implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &typeAssertMockContainer{} From e79e05b7bdc708afe12a5b2fcc245a3d57a491d4 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 16 Sep 2022 11:43:30 +0200 Subject: [PATCH 31/38] revert change --- pkg/runner/step_run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/step_run.go b/pkg/runner/step_run.go index 6464b7e7225..85fbc1b2b27 100644 --- a/pkg/runner/step_run.go +++ b/pkg/runner/step_run.go @@ -30,7 +30,7 @@ func (sr *stepRun) main() common.Executor { return runStepExecutor(sr, stepStageMain, common.NewPipelineExecutor( sr.setupShellCommandExecutor(), func(ctx context.Context) error { - return sr.getRunContext().JobContainer.Exec(sr.cmd, *sr.getEnv(), "", sr.Step.WorkingDirectory)(ctx) + return sr.getRunContext().JobContainer.Exec(sr.cmd, sr.env, "", sr.Step.WorkingDirectory)(ctx) }, )) } From 924fad83281c807f847d0fa429d0cb93c6db3ad0 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sun, 2 Oct 2022 21:57:12 +0200 Subject: [PATCH 32/38] Add original lookpath LICENSE --- pkg/lookpath/LICENSE | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 pkg/lookpath/LICENSE diff --git a/pkg/lookpath/LICENSE b/pkg/lookpath/LICENSE new file mode 100644 index 00000000000..ea5ea898692 --- /dev/null +++ b/pkg/lookpath/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file From a8677dd99e56a7fc8f5f98e71e2afa70c6e301d2 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sun, 2 Oct 2022 22:06:00 +0200 Subject: [PATCH 33/38] Avoid weird lint error --- pkg/lookpath/LICENSE | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/lookpath/LICENSE b/pkg/lookpath/LICENSE index ea5ea898692..83403eff153 100644 --- a/pkg/lookpath/LICENSE +++ b/pkg/lookpath/LICENSE @@ -4,13 +4,13 @@ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -24,4 +24,4 @@ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 1e8c1f69081a57bbe81c81f2c3fbcf5ee8eb5176 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sun, 2 Oct 2022 22:12:47 +0200 Subject: [PATCH 34/38] Remove disabled Test --- pkg/runner/expression_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/runner/expression_test.go b/pkg/runner/expression_test.go index 4363d72de73..283b6cf989e 100644 --- a/pkg/runner/expression_test.go +++ b/pkg/runner/expression_test.go @@ -117,8 +117,6 @@ func TestEvaluateRunContext(t *testing.T) { {"github.run_id", "1", ""}, {"github.run_number", "1", ""}, {"job.status", "success", ""}, - // disabled the os is not known without executionenvironment - // {"runner.os", "Linux", ""}, {"matrix.os", "Linux", ""}, {"matrix.foo", "bar", ""}, {"env.key", "value", ""}, From 18a4da83292094a195ad4568b0c84ec4e11f19b4 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sun, 9 Oct 2022 22:18:30 +0200 Subject: [PATCH 35/38] Fix merge --- pkg/runner/run_context.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 362609fff87..ce2532fa625 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -210,7 +210,6 @@ func (rc *RunContext) startJobContainer() common.Executor { } return true }) - hostname := rc.hostname(ctx) username, password, err := rc.handleCredentials(ctx) if err != nil { From 821c56efe52db58d708650a4953f780bc32aac5a Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 19 Oct 2022 22:28:04 +0200 Subject: [PATCH 36/38] Rewrite UpdateFromEnv --- pkg/container/host_environment.go | 50 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index 60911e66245..db332036877 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -341,15 +341,7 @@ func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[st } } -var _singleLineEnvPattern *regexp.Regexp -var _multiLineEnvPattern *regexp.Regexp - func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor { - if _singleLineEnvPattern == nil { - _singleLineEnvPattern = regexp.MustCompile(`^([^=]*)\=(.*)$`) - _multiLineEnvPattern = regexp.MustCompile(`^([^<]+)<<(\w+)$`) - } - localEnv := *env return func(ctx context.Context) error { envTar, err := e.GetContainerArchive(ctx, srcPath) @@ -363,27 +355,33 @@ func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) return err } s := bufio.NewScanner(reader) - multiLineEnvKey := "" - multiLineEnvDelimiter := "" - multiLineEnvContent := "" for s.Scan() { line := s.Text() - if singleLineEnv := _singleLineEnvPattern.FindStringSubmatch(line); singleLineEnv != nil { - localEnv[singleLineEnv[1]] = singleLineEnv[2] - } - if line == multiLineEnvDelimiter { - localEnv[multiLineEnvKey] = multiLineEnvContent - multiLineEnvKey, multiLineEnvDelimiter, multiLineEnvContent = "", "", "" - } - if multiLineEnvKey != "" && multiLineEnvDelimiter != "" { - if multiLineEnvContent != "" { - multiLineEnvContent += "\n" + singleLineEnv := strings.Index(line, "=") + multiLineEnv := strings.Index(line, "<<") + if singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv) { + localEnv[line[:singleLineEnv]] = line[singleLineEnv+1:] + } else if multiLineEnv != -1 { + multiLineEnvContent := "" + multiLineEnvDelimiter := line[multiLineEnv+2:] + delimiterFound := false + for s.Scan() { + content := s.Text() + if content == multiLineEnvDelimiter { + delimiterFound = true + break + } + if multiLineEnvContent != "" { + multiLineEnvContent += "\n" + } + multiLineEnvContent += content } - multiLineEnvContent += line - } - if mulitiLineEnvStart := _multiLineEnvPattern.FindStringSubmatch(line); mulitiLineEnvStart != nil { - multiLineEnvKey = mulitiLineEnvStart[1] - multiLineEnvDelimiter = mulitiLineEnvStart[2] + if !delimiterFound { + return fmt.Errorf("invalid format delimiter '%v' not found before end of file", ) + } + localEnv[line[:multiLineEnv]] = multiLineEnvContent + } else { + return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line) } } env = &localEnv From 618bff5d492df34b9b67be3a67bb90b13fe4a234 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 19 Oct 2022 22:31:13 +0200 Subject: [PATCH 37/38] Remove unused import --- pkg/container/host_environment.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index db332036877..7781c520b17 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -11,7 +11,6 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "runtime" "strings" "time" From 98f846aadb3b97f4631465152bcb61f4381c918c Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 19 Oct 2022 22:34:13 +0200 Subject: [PATCH 38/38] fix error message --- pkg/container/host_environment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index 7781c520b17..b404e86dd2a 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -376,7 +376,7 @@ func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) multiLineEnvContent += content } if !delimiterFound { - return fmt.Errorf("invalid format delimiter '%v' not found before end of file", ) + return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter) } localEnv[line[:multiLineEnv]] = multiLineEnvContent } else {