feat: add WASM transformer

Makefile

@@ -102,7 +102,7 @@ build: get $(BINDIR)/$(BINNAME) ## Build go code
 	@printf "\033[32m-------------------------------------\n BUILD SUCCESS\n-------------------------------------\033[0m\n"
 
 $(BINDIR)/$(BINNAME): $(SRC) $(ASSETS) $(WEB_ASSETS)
-	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o $(BINDIR)/$(BINNAME) .
+	CGO_ENABLED=1 go build -ldflags '$(LDFLAGS)' -o $(BINDIR)/$(BINNAME) .
 	mkdir -p $(GOPATH)/bin/
 	cp $(BINDIR)/$(BINNAME) $(GOPATH)/bin/
 
@@ -165,7 +165,7 @@ $(GOX):
 
 .PHONY: build-cross
 build-cross: $(GOX) clean $(SRC) $(ASSETS) $(WEB_ASSETS)
-	CGO_ENABLED=0 $(GOX) -parallel=3 -output="$(DISTDIR)/{{.OS}}-{{.Arch}}/$(BINNAME)" -osarch='$(TARGETS)' -ldflags '$(LDFLAGS)' .
+	CGO_ENABLED=1 $(GOX) -parallel=3 -output="$(DISTDIR)/{{.OS}}-{{.Arch}}/$(BINNAME)" -osarch='$(TARGETS)' -ldflags '$(LDFLAGS)' .
 
 .PHONY: dist
 dist: clean build-cross ## Build distribution

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/docker/cli v23.0.3+incompatible
 	github.com/docker/docker v23.0.3+incompatible
 	github.com/docker/libcompose v0.4.1-0.20171025083809-57bd716502dc
-	github.com/go-git/go-billy/v5 v5.3.1
+	github.com/go-git/go-billy/v5 v5.4.1
 	github.com/go-git/go-git/v5 v5.7.0
 	github.com/gobwas/glob v0.2.3
 	github.com/google/go-cmp v0.5.9
@@ -35,6 +35,7 @@ require (
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/pkg/errors v0.9.1
 	github.com/qri-io/starlib v0.5.0
+	github.com/second-state/WasmEdge-go v0.13.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cast v1.5.0
 	github.com/spf13/cobra v1.7.0
@@ -116,7 +117,6 @@ require (
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
-	github.com/go-git/go-billy/v5 v5.4.1 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect

go.sum

@@ -1823,6 +1823,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
+github.com/second-state/WasmEdge-go v0.13.0 h1:lCirXbSeqqvLLI67e330+F65EhkbvtAi7/ib913+sMs=
+github.com/second-state/WasmEdge-go v0.13.0/go.mod h1:HyBf9hVj1sRAjklsjc1Yvs9b5RcmthPG9z99dY78TKg=
 github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7AzSq3/kywjUDxSNq2SJ27RxCz2un0H3ePqE=
 github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989/go.mod h1:i9l/TNj+yDFh9SZXUTvspXTjbFXgZGP/UvhU1S65A4A=
 github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME=

transformer/external/wasmtransformer.go

@@ -0,0 +1,247 @@
+/*
+ *  Copyright IBM Corporation 2021
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package external
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/konveyor/move2kube/common"
+	"github.com/konveyor/move2kube/environment"
+	transformertypes "github.com/konveyor/move2kube/types/transformer"
+	"github.com/second-state/WasmEdge-go/wasmedge"
+	"github.com/sirupsen/logrus"
+	core "k8s.io/kubernetes/pkg/apis/core"
+)
+
+const (
+	wasmEnvDelimiter              = "="
+	detectInputPathWASMEnvKey     = "M2K_DETECT_INPUT_PATH"
+	detectOutputPathWASMEnvKey    = "M2K_DETECT_OUTPUT_PATH"
+	transformInputPathWASMEnvKey  = "M2K_TRANSFORM_INPUT_PATH"
+	transformOutputPathWASMEnvKey = "M2K_TRANSFORM_OUTPUT_PATH"
+)
+
+// WASM implements wasm transformer interface and is used for wasm based transformers
+type WASM struct {
+	Config     transformertypes.Transformer
+	Env        *environment.Environment
+	WASMConfig *WASMYamlConfig
+}
+
+// WASMYamlConfig is the format of wasm transformer yaml config
+type WASMYamlConfig struct {
+	WASMModule string        `yaml:"wasm_module"`
+	EnvList    []core.EnvVar `yaml:"env,omitempty"`
+}
+
+// Init Initializes the transformer
+func (t *WASM) Init(tc transformertypes.Transformer, env *environment.Environment) (err error) {
+	t.Config = tc
+	t.Env = env
+	t.WASMConfig = &WASMYamlConfig{}
+	if err := common.GetObjFromInterface(t.Config.Spec.Config, t.WASMConfig); err != nil {
+		return fmt.Errorf("unable to load config for Transformer %+v into %T . Error: %w", t.Config.Spec.Config, t.WASMConfig, err)
+	}
+
+	return nil
+}
+
+func (t *WASM) prepareEnv() []string {
+	envList := []string{}
+	for _, env := range t.WASMConfig.EnvList {
+		envList = append(envList, env.Name+wasmEnvDelimiter+env.Value)
+	}
+	return envList
+}
+
+// GetConfig returns the transformer config
+func (t *WASM) GetConfig() (transformertypes.Transformer, *environment.Environment) {
+	return t.Config, t.Env
+}
+
+// DirectoryDetect runs detect in each sub directory
+func (t *WASM) DirectoryDetect(dir string) (map[string][]transformertypes.Artifact, error) {
+	vm, err := t.initVm([]string{dir + ":" + dir})
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize WASM VM: %w", err)
+	}
+
+	allocateResult, err := vm.Execute("malloc", int32(len(dir)+1))
+	if err != nil {
+		return nil, fmt.Errorf("failed to alloc memory for directory: %w", err)
+	}
+	dirPointer := allocateResult[0].(int32)
+	mod := vm.GetActiveModule()
+	mem := mod.FindMemory("memory")
+	memData, err := mem.GetData(uint(dirPointer), uint(len(dir)+1))
+	if err != nil {
+		return nil, fmt.Errorf("failed to load wasm memory region: %w", err)
+	}
+	copy(memData, dir)
+	memData[len(dir)] = 0
+	directoryDetectOutput, dderr := vm.Execute("directoryDetect", dirPointer)
+	if dderr != nil {
+		err = fmt.Errorf("failed to execute directoryDetect in the wasm module. Error : %w", dderr)
+		return nil, err
+	}
+	directoryDetectOutputPointer := directoryDetectOutput[0].(int32)
+	memData, err = mem.GetData(uint(directoryDetectOutputPointer), 8)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load directoryDetect output: %w", err)
+	}
+	resultPointer := binary.LittleEndian.Uint32(memData[:4])
+	resultLength := binary.LittleEndian.Uint32(memData[4:])
+	memData, err = mem.GetData(uint(resultPointer), uint(resultLength))
+	if err != nil {
+		return nil, fmt.Errorf("failed to read directoryDetect output: %w", err)
+	}
+
+	services := map[string][]transformertypes.Artifact{}
+	err = json.Unmarshal(memData, &services)
+	if err != nil {
+		err = fmt.Errorf("failed to unmarshal directoryDetect output: %w", err)
+	}
+	return services, err
+}
+
+// Transform transforms the artifacts
+func (t *WASM) Transform(newArtifacts []transformertypes.Artifact, alreadySeenArtifacts []transformertypes.Artifact) ([]transformertypes.PathMapping, []transformertypes.Artifact, error) {
+	pathMappings := []transformertypes.PathMapping{}
+	createdArtifacts := []transformertypes.Artifact{}
+	data := make(map[string]interface{})
+	data["newArtifacts"] = newArtifacts
+	data["oldArtifacts"] = alreadySeenArtifacts
+	dataByt, err := json.Marshal(data)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to marshal transform input: %w", err)
+	}
+	dataStr := string(dataByt)
+
+	preopens := []string{}
+	for _, artifact := range newArtifacts {
+		for _, paths := range artifact.Paths {
+			for _, path := range paths {
+				preopens = append(preopens, path)
+			}
+		}
+	}
+
+	sort.Slice(preopens, func(i, j int) bool {
+		l1, l2 := len(preopens[i]), len(preopens[j])
+		if l1 != l2 {
+			return l1 < l2
+		}
+		return preopens[i] < preopens[j]
+	})
+
+	deduplicatedPreopens := []string{}
+	for _, path := range preopens {
+		shouldSkip := false
+		for _, existingPath := range deduplicatedPreopens {
+			if strings.HasPrefix(path, existingPath) {
+				shouldSkip = true
+				break
+			}
+		}
+
+		if !shouldSkip {
+			deduplicatedPreopens = append(deduplicatedPreopens, path)
+		}
+	}
+
+	finalPreopens := []string{}
+	for _, path := range deduplicatedPreopens {
+		finalPreopens = append(finalPreopens, path+":"+path)
+	}
+
+	vm, err := t.initVm(finalPreopens)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to initialize VM (transform): %w", err)
+	}
+
+	allocateResult, err := vm.Execute("malloc", int32(len(dataStr)+1))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to alloc memory for transform input: %w", err)
+	}
+	dataPointer := allocateResult[0].(int32)
+	mod := vm.GetActiveModule()
+	mem := mod.FindMemory("memory")
+	memData, err := mem.GetData(uint(dataPointer), uint(len(dataStr)+1))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to load wasm memory region: %w", err)
+	}
+	copy(memData, dataStr)
+	memData[len(dataStr)] = 0
+	transformOutput, err := vm.Execute("transform", dataPointer)
+	transformOutputPointer := transformOutput[0].(int32)
+	memData, err = mem.GetData(uint(transformOutputPointer), 8)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to load transform output: %w", err)
+	}
+	resultPointer := binary.LittleEndian.Uint32(memData[:4])
+	resultLength := binary.LittleEndian.Uint32(memData[4:])
+	memData, err = mem.GetData(uint(resultPointer), uint(resultLength))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to read transform output: %w", err)
+	}
+	logrus.Debug(string(memData))
+	var output transformertypes.TransformOutput
+	err = json.Unmarshal(memData, &output)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to unmarshal transformer output: %w", err)
+	}
+	pathMappings = append(pathMappings, output.PathMappings...)
+	createdArtifacts = append(createdArtifacts, output.CreatedArtifacts...)
+	return pathMappings, createdArtifacts, err
+}
+
+func (t *WASM) initVm(preopens []string) (*wasmedge.VM, error) {
+	wasmedge.SetLogErrorLevel()
+	conf := wasmedge.NewConfigure(wasmedge.WASI)
+	vm := wasmedge.NewVMWithConfig(conf)
+
+	wasi := vm.GetImportModule(wasmedge.WASI)
+	wasi.InitWasi(
+		[]string{},
+		t.prepareEnv(),
+		preopens,
+	)
+
+	err := vm.LoadWasmFile(filepath.Join(t.Env.GetEnvironmentContext(), t.WASMConfig.WASMModule))
+	if err != nil {
+		return nil, fmt.Errorf("failed to load wasm module %s: %w", t.WASMConfig.WASMModule, err)
+	}
+	err = vm.Validate()
+	if err != nil {
+		return nil, fmt.Errorf("failed to validate VM: %w", err)
+	}
+	err = vm.Instantiate()
+	if err != nil {
+		return nil, fmt.Errorf("failed to instantiate VM: %w", err)
+	}
+	_, err = vm.Execute("_start")
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute _start: %w", err)
+	}
+
+	return vm, nil
+}

transformer/transformer.go

@@ -88,6 +88,7 @@ var (
 
 func init() {
 	transformerObjs := []Transformer{
+		new(external.WASM),
 		new(external.Starlark),
 		new(external.Executable),