From 339e5b782229d7cc524998a6fdf22e2f66a6b4f6 Mon Sep 17 00:00:00 2001 From: Padmanabha V Seshadri <67090431+seshapad@users.noreply.github.com> Date: Tue, 29 Nov 2022 14:30:36 +0530 Subject: [PATCH] feat: Added containerized tranformer input/output configurability (#907) Signed-off-by: seshapad --- environment/environment.go | 6 +- environment/local.go | 3 +- environment/peercontainer.go | 3 +- transformer/cnbcontainerizer.go | 2 +- transformer/external/executabletransformer.go | 79 ++++++++++++++++--- 5 files changed, 77 insertions(+), 16 deletions(-) diff --git a/environment/environment.go b/environment/environment.go index dc7c24d5d..e022b6578 100644 --- a/environment/environment.go +++ b/environment/environment.go @@ -79,7 +79,7 @@ type EnvironmentInstance interface { Stat(name string) (fs.FileInfo, error) Download(envpath string) (outpath string, err error) Upload(outpath string) (envpath string, err error) - Exec(cmd environmenttypes.Command) (stdout string, stderr string, exitcode int, err error) + Exec(cmd environmenttypes.Command, envList []string) (stdout string, stderr string, exitcode int, err error) Destroy() error GetSource() string @@ -163,11 +163,11 @@ func (e *Environment) Reset() error { } // Exec executes an executable within the environment -func (e *Environment) Exec(cmd environmenttypes.Command) (stdout string, stderr string, exitcode int, err error) { +func (e *Environment) Exec(cmd environmenttypes.Command, envList []string) (stdout string, stderr string, exitcode int, err error) { if !e.active { return "", "", 0, ErrEnvironmentNotActive } - return e.Env.Exec(cmd) + return e.Env.Exec(cmd, envList) } // Destroy destroys all artifacts specific to the environment diff --git a/environment/local.go b/environment/local.go index 3c8b6f982..e140c5c8a 100644 --- a/environment/local.go +++ b/environment/local.go @@ -87,7 +87,7 @@ func (e *Local) Stat(name string) (fs.FileInfo, error) { } // Exec executes an executable within the environment -func (e *Local) Exec(cmd environmenttypes.Command) (stdout string, stderr string, exitcode int, err error) { +func (e *Local) Exec(cmd environmenttypes.Command, envList []string) (stdout string, stderr string, exitcode int, err error) { if common.DisableLocalExecution { return "", "", 0, fmt.Errorf("local execution prevented by %s flag", common.DisableLocalExecutionFlag) } @@ -102,6 +102,7 @@ func (e *Local) Exec(cmd environmenttypes.Command) (stdout string, stderr string execcmd.Stdout = &outb execcmd.Stderr = &errb execcmd.Env = e.getEnv() + execcmd.Env = append(execcmd.Env, envList...) if err := execcmd.Run(); err != nil { var ee *exec.ExitError var pe *os.PathError diff --git a/environment/peercontainer.go b/environment/peercontainer.go index e44bb247a..0ca61dd7b 100644 --- a/environment/peercontainer.go +++ b/environment/peercontainer.go @@ -137,7 +137,7 @@ func (e *PeerContainer) Stat(name string) (fs.FileInfo, error) { } // Exec executes a command in the container -func (e *PeerContainer) Exec(cmd environmenttypes.Command) (stdout string, stderr string, exitcode int, err error) { +func (e *PeerContainer) Exec(cmd environmenttypes.Command, envList []string) (stdout string, stderr string, exitcode int, err error) { cengine, err := container.GetContainerEngine(false) if err != nil { return "", "", 0, fmt.Errorf("failed to get the container engine. Error: %w", err) @@ -148,6 +148,7 @@ func (e *PeerContainer) Exec(cmd environmenttypes.Command) (stdout string, stder port := cast.ToString(e.GRPCQAReceiver.(*net.TCPAddr).Port) envs = append(envs, GRPCEnvName+"="+hostname+":"+port) } + envs = append(envs, envList...) return cengine.RunCmdInContainer(e.ContainerInfo.ID, cmd, e.ContainerInfo.WorkingDir, envs) } diff --git a/transformer/cnbcontainerizer.go b/transformer/cnbcontainerizer.go index e43086c6c..b661ffe90 100644 --- a/transformer/cnbcontainerizer.go +++ b/transformer/cnbcontainerizer.go @@ -88,7 +88,7 @@ func (t *CNBContainerizer) GetConfig() (transformertypes.Transformer, *environme func (t *CNBContainerizer) DirectoryDetect(dir string) (services map[string][]transformertypes.Artifact, err error) { path := dir cmd := environmenttypes.Command{"/cnb/lifecycle/detector", "-app", t.CNBEnv.Encode(path).(string)} - stdout, stderr, exitcode, err := t.CNBEnv.Exec(cmd) + stdout, stderr, exitcode, err := t.CNBEnv.Exec(cmd, nil) if err != nil { logrus.Debugf("CNB detector failed. exit code: %d error: %q\nstdout: %s\nstderr: %s", exitcode, err, stdout, stderr) return nil, fmt.Errorf("CNB detector failed with exitcode %d . Error: %q", exitcode, err) diff --git a/transformer/external/executabletransformer.go b/transformer/external/executabletransformer.go index f28a6eae9..b8c2f10d2 100644 --- a/transformer/external/executabletransformer.go +++ b/transformer/external/executabletransformer.go @@ -21,6 +21,7 @@ import ( "net" "os" "path/filepath" + "regexp" "github.com/dchest/uniuri" "github.com/konveyor/move2kube/common" @@ -32,6 +33,14 @@ import ( "github.com/sirupsen/logrus" ) +const ( + envDelimiter = "=" + detectInputPathEnvKey = "M2K_DETECT_INPUT_PATH" + detectOutputPathEnvKey = "M2K_DETECT_OUTPUT_PATH" + transformInputPathEnvKey = "M2K_TRANSFORM_INPUT_PATH" + transformOutputPathEnvKey = "M2K_TRANSFORM_OUTPUT_PATH" +) + // Executable implements transformer interface and is used to write simple external transformers type Executable struct { Config transformertypes.Transformer @@ -103,6 +112,7 @@ func (t *Executable) DirectoryDetect(dir string) (services map[string][]transfor services, err = t.executeDetect( t.ExecConfig.DirectoryDetectCMD, filepath.Join(containerInputDir, detectInputFile), + filepath.Join(detectContainerOutputDir, detectOutputFile), ) if err != nil { return services, fmt.Errorf("failed to execute the detect script. Error: %w", err) @@ -138,7 +148,16 @@ func (t *Executable) Transform(newArtifacts []transformertypes.Artifact, already if err != nil { return nil, nil, fmt.Errorf("failed to upload the transform input into the environment at the path '%s' . Error: %w", transformInputFile, err) } - stdout, stderr, exitcode, err := t.Env.Exec(append(t.ExecConfig.TransformCMD, filepath.Join(containerInputDir, transformInputFile))) + transformInputPath := filepath.Join(containerInputDir, transformInputFile) + transformOutputPath := filepath.Join(transformContainerOutputDir, transformOutputFile) + cmdToRun, envList := t.configIO( + t.ExecConfig.TransformCMD, + map[string]string{ + transformInputPathEnvKey: transformInputPath, + transformOutputPathEnvKey: transformOutputPath, + }, + ) + stdout, stderr, exitcode, err := t.Env.Exec(cmdToRun, envList) if err != nil { return nil, nil, fmt.Errorf("failed to run the transform.\nstdout: %s\nstderr: %s\nexit code: %d . Error: %w", stdout, stderr, exitcode, err) } @@ -161,21 +180,33 @@ func (t *Executable) Transform(newArtifacts []transformertypes.Artifact, already return pathMappings, createdArtifacts, nil } -func (t *Executable) executeDetect(cmd environmenttypes.Command, dir string) (services map[string][]transformertypes.Artifact, err error) { - stdout, stderr, exitcode, err := t.Env.Exec(append(cmd, dir)) +func (t *Executable) executeDetect( + cmd environmenttypes.Command, + inputPath string, + outputPath string, +) (services map[string][]transformertypes.Artifact, err error) { + cmdToRun, envList := t.configIO( + t.ExecConfig.DirectoryDetectCMD, + map[string]string{ + detectInputPathEnvKey: inputPath, + detectOutputPathEnvKey: outputPath, + }, + ) + stdout, stderr, exitcode, err := t.Env.Exec(cmdToRun, envList) if err != nil { return nil, fmt.Errorf("failed to execute the command in the environment.\nstdout: %s\nstderr: %s\nexit code: %d\nError: %w", stdout, stderr, exitcode, err) } else if exitcode != 0 { return nil, fmt.Errorf("the detect command failed with a non-zero exit code.\nstdout: %s\nstderr: %s\nexit code: %d", stdout, stderr, exitcode) } - logrus.Debugf("The detect command for transformer '%s' succeeded in the directory '%s'\nstdout: %s\nstderr: %s", t.Config.Name, dir, stdout, stderr) - outputPath, err := t.Env.Env.Download(detectContainerOutputDir) + logrus.Debugf("%s Detect succeeded in %s : %s, %s, %d", + t.Config.Name, inputPath, stdout, stderr, exitcode) + outputPathFromContainer, err := t.Env.Env.Download(detectContainerOutputDir) if err != nil { - return nil, fmt.Errorf("failed to download the json output at path '%s' from the environment. Error: %w", outputPath, err) + return nil, fmt.Errorf("failed to download the json output at path '%s' from the environment. Error: %w", outputPathFromContainer, err) } - logrus.Debugf("Output detect JSON path: %v", outputPath) + logrus.Debugf("Output detect JSON path: %v", outputPathFromContainer) output := map[string][]transformertypes.Artifact{} - jsonOutputPath := filepath.Join(outputPath, detectOutputFile) + jsonOutputPath := filepath.Join(outputPathFromContainer, detectOutputFile) if err := common.ReadJSON(jsonOutputPath, &output); err != nil { logrus.Warnf("failed in unmarshal the detect output file at path '%s' as json. Trying with config type. Error: %q", jsonOutputPath, err) config := map[string]interface{}{} @@ -183,8 +214,10 @@ func (t *Executable) executeDetect(cmd environmenttypes.Command, dir string) (se logrus.Warnf("failed in unmarshal the detect output file at path '%s' as config json. Error: %q", jsonOutputPath, err) } trans := transformertypes.Artifact{ - Paths: map[transformertypes.PathType][]string{artifacts.ServiceDirPathType: {dir}}, - Configs: map[transformertypes.ConfigType]interface{}{TemplateConfigType: config}, + Paths: map[transformertypes.PathType][]string{artifacts.ServiceDirPathType: {inputPath}}, + Configs: map[transformertypes.ConfigType]interface{}{ + TemplateConfigType: config, + }, } return map[string][]transformertypes.Artifact{"": {trans}}, nil } @@ -204,3 +237,29 @@ func (t *Executable) uploadInput(data interface{}, inputFile string) (string, er } return containerInputDir, nil } + +func (t *Executable) configIO( + cmd environmenttypes.Command, + kvMap map[string]string) ( + environmenttypes.Command, + []string, +) { + bracketRegex := regexp.MustCompile(`[{\(\)}]`) + cmdToRun := cmd + for envKey, value := range kvMap { + for index, token := range cmdToRun { + if bracketRegex.Match([]byte(token)) { + regexExp := regexp.MustCompile(`\${` + envKey + `}|\$\(` + envKey + `\)`) + cmdToRun[index] = regexExp.ReplaceAllString(token, value) + } else { + regexExp := regexp.MustCompile(`\$` + envKey) + cmdToRun[index] = regexExp.ReplaceAllString(token, value) + } + } + } + envList := []string{} + for envKey, value := range kvMap { + envList = append(envList, envKey+envDelimiter+value) + } + return cmdToRun, envList +}