From 1a3e656242894fd2bd8a6727292f00abb9ac4d4e Mon Sep 17 00:00:00 2001 From: Soumil Paranjpay <82497827+Soumil-07@users.noreply.github.com> Date: Fri, 6 Oct 2023 11:41:09 -0700 Subject: [PATCH] feat: add handler for wasm transformer Signed-off-by: Soumil Paranjpay Based on work by Vamsi Krishna --- Makefile | 4 +- go.mod | 4 +- go.sum | 2 + transformer/external/wasmtransformer.go | 247 ++++++++++++++++++++++++ transformer/transformer.go | 1 + 5 files changed, 254 insertions(+), 4 deletions(-) create mode 100644 transformer/external/wasmtransformer.go diff --git a/Makefile b/Makefile index 262f18c0e..be3f5a415 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ build: get $(BINDIR)/$(BINNAME) ## Build go code @printf "\033[32m-------------------------------------\n BUILD SUCCESS\n-------------------------------------\033[0m\n" $(BINDIR)/$(BINNAME): $(SRC) $(ASSETS) $(WEB_ASSETS) - CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o $(BINDIR)/$(BINNAME) . + CGO_ENABLED=1 go build -ldflags '$(LDFLAGS)' -o $(BINDIR)/$(BINNAME) . mkdir -p $(GOPATH)/bin/ cp $(BINDIR)/$(BINNAME) $(GOPATH)/bin/ @@ -165,7 +165,7 @@ $(GOX): .PHONY: build-cross build-cross: $(GOX) clean $(SRC) $(ASSETS) $(WEB_ASSETS) - CGO_ENABLED=0 $(GOX) -parallel=3 -output="$(DISTDIR)/{{.OS}}-{{.Arch}}/$(BINNAME)" -osarch='$(TARGETS)' -ldflags '$(LDFLAGS)' . + CGO_ENABLED=1 $(GOX) -parallel=3 -output="$(DISTDIR)/{{.OS}}-{{.Arch}}/$(BINNAME)" -osarch='$(TARGETS)' -ldflags '$(LDFLAGS)' . .PHONY: dist dist: clean build-cross ## Build distribution diff --git a/go.mod b/go.mod index a58265781..08d329965 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/docker/cli v23.0.3+incompatible github.com/docker/docker v23.0.3+incompatible github.com/docker/libcompose v0.4.1-0.20171025083809-57bd716502dc - github.com/go-git/go-billy/v5 v5.3.1 + github.com/go-git/go-billy/v5 v5.4.1 github.com/go-git/go-git/v5 v5.7.0 github.com/gobwas/glob v0.2.3 github.com/google/go-cmp v0.5.9 @@ -35,6 +35,7 @@ require ( github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pkg/errors v0.9.1 github.com/qri-io/starlib v0.5.0 + github.com/second-state/WasmEdge-go v0.13.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.7.0 @@ -116,7 +117,6 @@ require ( github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.4.1 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.2.4 // indirect diff --git a/go.sum b/go.sum index ed86ce246..de8249eba 100644 --- a/go.sum +++ b/go.sum @@ -1823,6 +1823,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/second-state/WasmEdge-go v0.13.0 h1:lCirXbSeqqvLLI67e330+F65EhkbvtAi7/ib913+sMs= +github.com/second-state/WasmEdge-go v0.13.0/go.mod h1:HyBf9hVj1sRAjklsjc1Yvs9b5RcmthPG9z99dY78TKg= github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7AzSq3/kywjUDxSNq2SJ27RxCz2un0H3ePqE= github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989/go.mod h1:i9l/TNj+yDFh9SZXUTvspXTjbFXgZGP/UvhU1S65A4A= github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME= diff --git a/transformer/external/wasmtransformer.go b/transformer/external/wasmtransformer.go new file mode 100644 index 000000000..5eb853a3e --- /dev/null +++ b/transformer/external/wasmtransformer.go @@ -0,0 +1,247 @@ +/* + * Copyright IBM Corporation 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package external + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/konveyor/move2kube/common" + "github.com/konveyor/move2kube/environment" + transformertypes "github.com/konveyor/move2kube/types/transformer" + "github.com/second-state/WasmEdge-go/wasmedge" + "github.com/sirupsen/logrus" + core "k8s.io/kubernetes/pkg/apis/core" +) + +const ( + wasmEnvDelimiter = "=" + detectInputPathWASMEnvKey = "M2K_DETECT_INPUT_PATH" + detectOutputPathWASMEnvKey = "M2K_DETECT_OUTPUT_PATH" + transformInputPathWASMEnvKey = "M2K_TRANSFORM_INPUT_PATH" + transformOutputPathWASMEnvKey = "M2K_TRANSFORM_OUTPUT_PATH" +) + +// WASM implements wasm transformer interface and is used for wasm based transformers +type WASM struct { + Config transformertypes.Transformer + Env *environment.Environment + WASMConfig *WASMYamlConfig +} + +// WASMYamlConfig is the format of wasm transformer yaml config +type WASMYamlConfig struct { + WASMModule string `yaml:"wasm_module"` + EnvList []core.EnvVar `yaml:"env,omitempty"` +} + +// Init Initializes the transformer +func (t *WASM) Init(tc transformertypes.Transformer, env *environment.Environment) (err error) { + t.Config = tc + t.Env = env + t.WASMConfig = &WASMYamlConfig{} + if err := common.GetObjFromInterface(t.Config.Spec.Config, t.WASMConfig); err != nil { + return fmt.Errorf("unable to load config for Transformer %+v into %T . Error: %w", t.Config.Spec.Config, t.WASMConfig, err) + } + + return nil +} + +func (t *WASM) prepareEnv() []string { + envList := []string{} + for _, env := range t.WASMConfig.EnvList { + envList = append(envList, env.Name+wasmEnvDelimiter+env.Value) + } + return envList +} + +// GetConfig returns the transformer config +func (t *WASM) GetConfig() (transformertypes.Transformer, *environment.Environment) { + return t.Config, t.Env +} + +// DirectoryDetect runs detect in each sub directory +func (t *WASM) DirectoryDetect(dir string) (map[string][]transformertypes.Artifact, error) { + vm, err := t.initVm([]string{dir + ":" + dir}) + if err != nil { + return nil, fmt.Errorf("failed to initialize WASM VM: %w", err) + } + + allocateResult, err := vm.Execute("malloc", int32(len(dir)+1)) + if err != nil { + return nil, fmt.Errorf("failed to alloc memory for directory: %w", err) + } + dirPointer := allocateResult[0].(int32) + mod := vm.GetActiveModule() + mem := mod.FindMemory("memory") + memData, err := mem.GetData(uint(dirPointer), uint(len(dir)+1)) + if err != nil { + return nil, fmt.Errorf("failed to load wasm memory region: %w", err) + } + copy(memData, dir) + memData[len(dir)] = 0 + directoryDetectOutput, dderr := vm.Execute("directoryDetect", dirPointer) + if dderr != nil { + err = fmt.Errorf("failed to execute directoryDetect in the wasm module. Error : %w", dderr) + return nil, err + } + directoryDetectOutputPointer := directoryDetectOutput[0].(int32) + memData, err = mem.GetData(uint(directoryDetectOutputPointer), 8) + if err != nil { + return nil, fmt.Errorf("failed to load directoryDetect output: %w", err) + } + resultPointer := binary.LittleEndian.Uint32(memData[:4]) + resultLength := binary.LittleEndian.Uint32(memData[4:]) + memData, err = mem.GetData(uint(resultPointer), uint(resultLength)) + if err != nil { + return nil, fmt.Errorf("failed to read directoryDetect output: %w", err) + } + + services := map[string][]transformertypes.Artifact{} + err = json.Unmarshal(memData, &services) + if err != nil { + err = fmt.Errorf("failed to unmarshal directoryDetect output: %w", err) + } + return services, err +} + +// Transform transforms the artifacts +func (t *WASM) Transform(newArtifacts []transformertypes.Artifact, alreadySeenArtifacts []transformertypes.Artifact) ([]transformertypes.PathMapping, []transformertypes.Artifact, error) { + pathMappings := []transformertypes.PathMapping{} + createdArtifacts := []transformertypes.Artifact{} + data := make(map[string]interface{}) + data["newArtifacts"] = newArtifacts + data["oldArtifacts"] = alreadySeenArtifacts + dataByt, err := json.Marshal(data) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal transform input: %w", err) + } + dataStr := string(dataByt) + + preopens := []string{} + for _, artifact := range newArtifacts { + for _, paths := range artifact.Paths { + for _, path := range paths { + preopens = append(preopens, path) + } + } + } + + sort.Slice(preopens, func(i, j int) bool { + l1, l2 := len(preopens[i]), len(preopens[j]) + if l1 != l2 { + return l1 < l2 + } + return preopens[i] < preopens[j] + }) + + deduplicatedPreopens := []string{} + for _, path := range preopens { + shouldSkip := false + for _, existingPath := range deduplicatedPreopens { + if strings.HasPrefix(path, existingPath) { + shouldSkip = true + break + } + } + + if !shouldSkip { + deduplicatedPreopens = append(deduplicatedPreopens, path) + } + } + + finalPreopens := []string{} + for _, path := range deduplicatedPreopens { + finalPreopens = append(finalPreopens, path+":"+path) + } + + vm, err := t.initVm(finalPreopens) + if err != nil { + return nil, nil, fmt.Errorf("failed to initialize VM (transform): %w", err) + } + + allocateResult, err := vm.Execute("malloc", int32(len(dataStr)+1)) + if err != nil { + return nil, nil, fmt.Errorf("failed to alloc memory for transform input: %w", err) + } + dataPointer := allocateResult[0].(int32) + mod := vm.GetActiveModule() + mem := mod.FindMemory("memory") + memData, err := mem.GetData(uint(dataPointer), uint(len(dataStr)+1)) + if err != nil { + return nil, nil, fmt.Errorf("failed to load wasm memory region: %w", err) + } + copy(memData, dataStr) + memData[len(dataStr)] = 0 + transformOutput, err := vm.Execute("transform", dataPointer) + transformOutputPointer := transformOutput[0].(int32) + memData, err = mem.GetData(uint(transformOutputPointer), 8) + if err != nil { + return nil, nil, fmt.Errorf("failed to load transform output: %w", err) + } + resultPointer := binary.LittleEndian.Uint32(memData[:4]) + resultLength := binary.LittleEndian.Uint32(memData[4:]) + memData, err = mem.GetData(uint(resultPointer), uint(resultLength)) + if err != nil { + return nil, nil, fmt.Errorf("failed to read transform output: %w", err) + } + logrus.Debug(string(memData)) + var output transformertypes.TransformOutput + err = json.Unmarshal(memData, &output) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal transformer output: %w", err) + } + pathMappings = append(pathMappings, output.PathMappings...) + createdArtifacts = append(createdArtifacts, output.CreatedArtifacts...) + return pathMappings, createdArtifacts, err +} + +func (t *WASM) initVm(preopens []string) (*wasmedge.VM, error) { + wasmedge.SetLogErrorLevel() + conf := wasmedge.NewConfigure(wasmedge.WASI) + vm := wasmedge.NewVMWithConfig(conf) + + wasi := vm.GetImportModule(wasmedge.WASI) + wasi.InitWasi( + []string{}, + t.prepareEnv(), + preopens, + ) + + err := vm.LoadWasmFile(filepath.Join(t.Env.GetEnvironmentContext(), t.WASMConfig.WASMModule)) + if err != nil { + return nil, fmt.Errorf("failed to load wasm module %s: %w", t.WASMConfig.WASMModule, err) + } + err = vm.Validate() + if err != nil { + return nil, fmt.Errorf("failed to validate VM: %w", err) + } + err = vm.Instantiate() + if err != nil { + return nil, fmt.Errorf("failed to instantiate VM: %w", err) + } + _, err = vm.Execute("_start") + if err != nil { + return nil, fmt.Errorf("failed to execute _start: %w", err) + } + + return vm, nil +} diff --git a/transformer/transformer.go b/transformer/transformer.go index 87b7aa1d6..058bc85ba 100644 --- a/transformer/transformer.go +++ b/transformer/transformer.go @@ -88,6 +88,7 @@ var ( func init() { transformerObjs := []Transformer{ + new(external.WASM), new(external.Starlark), new(external.Executable),