Add support for local Wasm plugins (#3349)

CHANGELOG.md

@@ -6,6 +6,8 @@
   they satisfy the field constraints, and are only present if constraints are present.
 - Update the `PROTOVALIDATE` lint rule to check predefined rules. Predefined rules will be checked
   that they compile.
+- Add support for a WebAssembly (Wasm) runtime for custom lint and breaking changes plugins. Use the
+  `.wasm` file extension to specify a path to a Wasm plugin.
 
 ## [v1.43.0] - 2024-09-30
 

go.mod

@@ -35,6 +35,7 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
+	github.com/tetratelabs/wazero v1.8.0
 	go.lsp.dev/jsonrpc2 v0.10.0
 	go.lsp.dev/protocol v0.12.0
 	go.opentelemetry.io/otel v1.30.0

go.sum

@@ -252,6 +252,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g=
+github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
 github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23envGs=
 github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

make/buf/all.mk

@@ -24,6 +24,8 @@ GO_TEST_BINS := $(GO_TEST_BINS) \
 	private/bufpkg/bufcheck/internal/cmd/buf-plugin-rpc-ext \
 	private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-category \
 	private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-rule
+GO_TEST_WASM_BINS := $(GO_TEST_WASM_BINS) \
+	private/bufpkg/bufcheck/internal/cmd/buf-plugin-suffix
 GO_MOD_VERSION := 1.22
 DOCKER_BINS := $(DOCKER_BINS) buf
 FILE_IGNORES := $(FILE_IGNORES) \

make/go/go.mk

@@ -16,6 +16,8 @@ GO_BINS ?=
 # Settable
 GO_TEST_BINS ?=
 # Settable
+GO_TEST_WASM_BINS ?=
+# Settable
 GO_GET_PKGS ?=
 # Settable
 GO_MOD_VERSION ?= 1.22
@@ -142,7 +144,7 @@ build: prebuild ## Run go build.
 pretest::
 
 .PHONY: test
-test: pretest installtest ## Run all go tests.
+test: pretest installtest installtestwasm ## Run all go tests.
 	go test $(GO_TEST_FLAGS) $(GOPKGS)
 
 .PHONY: testrace
@@ -203,3 +205,17 @@ endef
 
 $(foreach gobin,$(sort $(GO_TEST_BINS)),$(eval $(call gotestbinfunc,$(gobin))))
 $(foreach gobin,$(sort $(GO_TEST_BINS)),$(eval FILE_IGNORES := $(FILE_IGNORES) $(gobin)/$(notdir $(gobin))))
+
+.PHONY: installtestwasm
+installtestwasm::
+
+define gotestwasmfunc
+.PHONY: installtestwasm$(notdir $(1))
+installtestwasm$(notdir $(1)):
+	GOOS=wasip1 GOARCH=wasm go build -o $(GOBIN)/$(notdir $(1)).wasm ./$(1)
+
+installtestwasm:: installtestwasm$(notdir $(1))
+endef
+
+$(foreach gobin,$(sort $(GO_TEST_WASM_BINS)),$(eval $(call gotestwasmfunc,$(gobin))))
+$(foreach gobin,$(sort $(GO_TEST_WASM_BINS)),$(eval FILE_IGNORES := $(FILE_IGNORES) $(gobin)/$(notdir $(gobin))))

private/buf/bufcli/cache.go

@@ -103,6 +103,11 @@ var (
 	//
 	// Normalized.
 	v3CacheModuleLockRelDirPath = normalpath.Join("v3", "modulelocks")
+	// v3CacheWasmRuntimeRelDirPath is the relative path to the Wasm runtime cache directory in its newest iteration.
+	// This directory is used to store the Wasm runtime cache. This is an implementation specific cache and opaque outside of the runtime.
+	//
+	// Normalized.
+	v3CacheWasmRuntimeRelDirPath = normalpath.Join("v3", "wasmruntime")
 )
 
 // NewModuleDataProvider returns a new ModuleDataProvider while creating the
@@ -135,6 +140,19 @@ func NewCommitProvider(container appext.Container) (bufmodule.CommitProvider, er
 	)
 }
 
+// CreateWasmRuntimeCacheDir creates the cache directory for the Wasm runtime.
+//
+// This is used by the Wasm runtime to cache compiled Wasm plugins. This is an
+// implementation specific cache and opaque outside of the runtime. The runtime
+// will manage the cache versioning itself within this directory.
+func CreateWasmRuntimeCacheDir(container appext.Container) (string, error) {
+	if err := createCacheDir(container.CacheDirPath(), v3CacheWasmRuntimeRelDirPath); err != nil {
+		return "", err
+	}
+	fullCacheDirPath := normalpath.Join(container.CacheDirPath(), v3CacheWasmRuntimeRelDirPath)
+	return fullCacheDirPath, nil
+}
+
 // newWKTStore returns a new bufwktstore.Store while creating the required cache directories.
 func newWKTStore(container appext.Container) (bufwktstore.Store, error) {
 	if err := createCacheDir(container.CacheDirPath(), v3CacheWKTRelDirPath); err != nil {

private/buf/buflsp/buflsp.go

@@ -26,7 +26,6 @@ import (
 	"github.com/bufbuild/buf/private/bufpkg/bufcheck"
 	"github.com/bufbuild/buf/private/bufpkg/bufimage"
 	"github.com/bufbuild/buf/private/pkg/app/appext"
-	"github.com/bufbuild/buf/private/pkg/command"
 	"github.com/bufbuild/buf/private/pkg/storage"
 	"github.com/bufbuild/buf/private/pkg/storage/storageos"
 	"github.com/bufbuild/buf/private/pkg/tracing"
@@ -43,6 +42,7 @@ func Serve(
 	ctx context.Context,
 	container appext.Container,
 	controller bufctl.Controller,
+	checkClient bufcheck.Client,
 	stream jsonrpc2.Stream,
 ) (jsonrpc2.Conn, error) {
 	// The LSP protocol deals with absolute filesystem paths. This requires us to
@@ -57,12 +57,6 @@ func Serve(
 		return nil, err
 	}
 
-	tracer := tracing.NewTracer(container.Tracer())
-	checkClient, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
-	if err != nil {
-		return nil, err
-	}
-
 	conn := jsonrpc2.NewConn(stream)
 	lsp := &lsp{
 		conn: conn,
@@ -71,7 +65,7 @@ func Serve(
 			zap.NewNop(), // The logging from protocol itself isn't very good, we've replaced it with connAdapter here.
 		),
 		logger:      container.Logger(),
-		tracer:      tracer,
+		tracer:      tracing.NewTracer(container.Tracer()),
 		controller:  controller,
 		checkClient: checkClient,
 		rootBucket:  bucket,

private/buf/bufmigrate/migrator.go

@@ -32,6 +32,7 @@ import (
 	"github.com/bufbuild/buf/private/pkg/storage"
 	"github.com/bufbuild/buf/private/pkg/storage/storagemem"
 	"github.com/bufbuild/buf/private/pkg/tracing"
+	"github.com/bufbuild/buf/private/pkg/wasm"
 	"github.com/google/uuid"
 	"go.uber.org/multierr"
 	"go.uber.org/zap"
@@ -712,7 +713,7 @@ func equivalentCheckConfigInV2(
 ) (bufconfig.CheckConfig, error) {
 	// No need for custom lint/breaking plugins since there's no plugins to migrate from <=v1.
 	// TODO: If we ever need v3, then we will have to deal with this.
-	client, err := bufcheck.NewClient(logger, tracer, bufcheck.NewRunnerProvider(runner))
+	client, err := bufcheck.NewClient(logger, tracer, bufcheck.NewRunnerProvider(runner, wasm.UnimplementedRuntime))
 	if err != nil {
 		return nil, err
 	}

private/buf/cmd/buf/buf_test.go

@@ -45,6 +45,7 @@ import (
 	"github.com/bufbuild/buf/private/pkg/storage/storageos"
 	"github.com/bufbuild/buf/private/pkg/storage/storagetesting"
 	"github.com/bufbuild/buf/private/pkg/tracing"
+	"github.com/bufbuild/buf/private/pkg/wasm"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
@@ -1349,7 +1350,7 @@ func TestCheckLsBreakingRulesFromConfigExceptDeprecated(t *testing.T) {
 		t.Run(version.String(), func(t *testing.T) {
 			t.Parallel()
 			// Do not need any custom lint/breaking plugins here.
-			client, err := bufcheck.NewClient(zap.NewNop(), tracing.NopTracer, bufcheck.NewRunnerProvider(command.NewRunner()))
+			client, err := bufcheck.NewClient(zap.NewNop(), tracing.NopTracer, bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime))
 			require.NoError(t, err)
 			allRules, err := client.AllRules(context.Background(), check.RuleTypeBreaking, version)
 			require.NoError(t, err)

private/buf/cmd/buf/command/beta/lsp/lsp.go

@@ -25,11 +25,16 @@ import (
 
 	"github.com/bufbuild/buf/private/buf/bufcli"
 	"github.com/bufbuild/buf/private/buf/buflsp"
+	"github.com/bufbuild/buf/private/bufpkg/bufcheck"
 	"github.com/bufbuild/buf/private/pkg/app/appcmd"
 	"github.com/bufbuild/buf/private/pkg/app/appext"
+	"github.com/bufbuild/buf/private/pkg/command"
 	"github.com/bufbuild/buf/private/pkg/ioext"
+	"github.com/bufbuild/buf/private/pkg/tracing"
+	"github.com/bufbuild/buf/private/pkg/wasm"
 	"github.com/spf13/pflag"
 	"go.lsp.dev/jsonrpc2"
+	"go.uber.org/multierr"
 )
 
 const (
@@ -77,7 +82,7 @@ func run(
 	ctx context.Context,
 	container appext.Container,
 	flags *flags,
-) error {
+) (retErr error) {
 	bufcli.WarnBetaCommand(ctx, container)
 
 	transport, err := dial(container, flags)
@@ -90,7 +95,28 @@ func run(
 		return err
 	}
 
-	conn, err := buflsp.Serve(ctx, container, controller, jsonrpc2.NewStream(transport))
+	wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container)
+	if err != nil {
+		return err
+	}
+	wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		retErr = multierr.Append(retErr, wasmRuntime.Close(ctx))
+	}()
+	checkClient, err := bufcheck.NewClient(
+		container.Logger(),
+		tracing.NewTracer(container.Tracer()),
+		bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime),
+		bufcheck.ClientWithStderr(container.Stderr()),
+	)
+	if err != nil {
+		return err
+	}
+
+	conn, err := buflsp.Serve(ctx, container, controller, checkClient, jsonrpc2.NewStream(transport))
 	if err != nil {
 		return err
 	}

private/buf/cmd/buf/command/breaking/breaking.go

@@ -31,7 +31,9 @@ import (
 	"github.com/bufbuild/buf/private/pkg/slicesext"
 	"github.com/bufbuild/buf/private/pkg/stringutil"
 	"github.com/bufbuild/buf/private/pkg/tracing"
+	"github.com/bufbuild/buf/private/pkg/wasm"
 	"github.com/spf13/pflag"
+	"go.uber.org/multierr"
 )
 
 const (
@@ -145,7 +147,7 @@ func run(
 	ctx context.Context,
 	container appext.Container,
 	flags *flags,
-) error {
+) (retErr error) {
 	if err := bufcli.ValidateRequiredFlag(againstFlagName, flags.Against); err != nil {
 		return err
 	}
@@ -206,10 +208,26 @@ func run(
 			len(againstImageWithConfigs),
 		)
 	}
+	wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container)
+	if err != nil {
+		return err
+	}
+	wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		retErr = multierr.Append(retErr, wasmRuntime.Close(ctx))
+	}()
 	tracer := tracing.NewTracer(container.Tracer())
 	var allFileAnnotations []bufanalysis.FileAnnotation
 	for i, imageWithConfig := range imageWithConfigs {
-		client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
+		client, err := bufcheck.NewClient(
+			container.Logger(),
+			tracer,
+			bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime),
+			bufcheck.ClientWithStderr(container.Stderr()),
+		)
 		if err != nil {
 			return err
 		}

private/buf/cmd/buf/command/config/internal/internal.go

@@ -32,7 +32,9 @@ import (
 	"github.com/bufbuild/buf/private/pkg/stringutil"
 	"github.com/bufbuild/buf/private/pkg/syserror"
 	"github.com/bufbuild/buf/private/pkg/tracing"
+	"github.com/bufbuild/buf/private/pkg/wasm"
 	"github.com/spf13/pflag"
+	"go.uber.org/multierr"
 )
 
 const (
@@ -149,7 +151,7 @@ func lsRun(
 	flags *flags,
 	commandName string,
 	ruleType check.RuleType,
-) error {
+) (retErr error) {
 	if flags.ConfiguredOnly {
 		if flags.Version != "" {
 			return appcmd.NewInvalidArgumentErrorf("--%s cannot be specified if --%s is specified", versionFlagName, configFlagName)
@@ -184,8 +186,24 @@ func lsRun(
 			return err
 		}
 	}
+	wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container)
+	if err != nil {
+		return err
+	}
+	wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		retErr = multierr.Append(retErr, wasmRuntime.Close(ctx))
+	}()
 	tracer := tracing.NewTracer(container.Tracer())
-	client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
+	client, err := bufcheck.NewClient(
+		container.Logger(),
+		tracer,
+		bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime),
+		bufcheck.ClientWithStderr(container.Stderr()),
+	)
 	if err != nil {
 		return err
 	}

private/buf/cmd/buf/command/lint/lint.go

@@ -28,7 +28,9 @@ import (
 	"github.com/bufbuild/buf/private/pkg/command"
 	"github.com/bufbuild/buf/private/pkg/stringutil"
 	"github.com/bufbuild/buf/private/pkg/tracing"
+	"github.com/bufbuild/buf/private/pkg/wasm"
 	"github.com/spf13/pflag"
+	"go.uber.org/multierr"
 )
 
 const (
@@ -131,10 +133,26 @@ func run(
 	if err != nil {
 		return err
 	}
+	wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container)
+	if err != nil {
+		return err
+	}
+	wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		retErr = multierr.Append(retErr, wasmRuntime.Close(ctx))
+	}()
 	tracer := tracing.NewTracer(container.Tracer())
 	var allFileAnnotations []bufanalysis.FileAnnotation
 	for _, imageWithConfig := range imageWithConfigs {
-		client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
+		client, err := bufcheck.NewClient(
+			container.Logger(),
+			tracer,
+			bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime),
+			bufcheck.ClientWithStderr(container.Stderr()),
+		)
 		if err != nil {
 			return err
 		}

private/buf/cmd/buf/command/mod/internal/internal.go

@@ -31,6 +31,7 @@ import (
 	"github.com/bufbuild/buf/private/pkg/stringutil"
 	"github.com/bufbuild/buf/private/pkg/syserror"
 	"github.com/bufbuild/buf/private/pkg/tracing"
+	"github.com/bufbuild/buf/private/pkg/wasm"
 	"github.com/spf13/pflag"
 )
 
@@ -175,7 +176,12 @@ func lsRun(
 	}
 	// BufYAMLFiles <=v1 never had plugins.
 	tracer := tracing.NewTracer(container.Tracer())
-	client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
+	client, err := bufcheck.NewClient(
+		container.Logger(),
+		tracer,
+		bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime),
+		bufcheck.ClientWithStderr(container.Stderr()),
+	)
 	if err != nil {
 		return err
 	}