diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d55799bc3..898414b646 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ they satisfy the field constraints, and are only present if constraints are present. - Update the `PROTOVALIDATE` lint rule to check predefined rules. Predefined rules will be checked that they compile. +- Add support for a WebAssembly (Wasm) runtime for custom lint and breaking changes plugins. Use the + `.wasm` file extension to specify a path to a Wasm plugin. ## [v1.43.0] - 2024-09-30 diff --git a/go.mod b/go.mod index f52bebe278..b7820598f7 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 + github.com/tetratelabs/wazero v1.8.0 go.lsp.dev/jsonrpc2 v0.10.0 go.lsp.dev/protocol v0.12.0 go.opentelemetry.io/otel v1.30.0 diff --git a/go.sum b/go.sum index 1290e4fafe..40219a81b5 100644 --- a/go.sum +++ b/go.sum @@ -252,6 +252,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g= +github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23envGs= github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/make/buf/all.mk b/make/buf/all.mk index eb21f80a76..07b9b45c37 100644 --- a/make/buf/all.mk +++ b/make/buf/all.mk @@ -24,6 +24,8 @@ GO_TEST_BINS := $(GO_TEST_BINS) \ private/bufpkg/bufcheck/internal/cmd/buf-plugin-rpc-ext \ private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-category \ private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-rule +GO_TEST_WASM_BINS := $(GO_TEST_WASM_BINS) \ + private/bufpkg/bufcheck/internal/cmd/buf-plugin-suffix GO_MOD_VERSION := 1.22 DOCKER_BINS := $(DOCKER_BINS) buf FILE_IGNORES := $(FILE_IGNORES) \ diff --git a/make/go/go.mk b/make/go/go.mk index 5efe195474..107d09f4c2 100644 --- a/make/go/go.mk +++ b/make/go/go.mk @@ -16,6 +16,8 @@ GO_BINS ?= # Settable GO_TEST_BINS ?= # Settable +GO_TEST_WASM_BINS ?= +# Settable GO_GET_PKGS ?= # Settable GO_MOD_VERSION ?= 1.22 @@ -142,7 +144,7 @@ build: prebuild ## Run go build. pretest:: .PHONY: test -test: pretest installtest ## Run all go tests. +test: pretest installtest installtestwasm ## Run all go tests. go test $(GO_TEST_FLAGS) $(GOPKGS) .PHONY: testrace @@ -203,3 +205,17 @@ endef $(foreach gobin,$(sort $(GO_TEST_BINS)),$(eval $(call gotestbinfunc,$(gobin)))) $(foreach gobin,$(sort $(GO_TEST_BINS)),$(eval FILE_IGNORES := $(FILE_IGNORES) $(gobin)/$(notdir $(gobin)))) + +.PHONY: installtestwasm +installtestwasm:: + +define gotestwasmfunc +.PHONY: installtestwasm$(notdir $(1)) +installtestwasm$(notdir $(1)): + GOOS=wasip1 GOARCH=wasm go build -o $(GOBIN)/$(notdir $(1)).wasm ./$(1) + +installtestwasm:: installtestwasm$(notdir $(1)) +endef + +$(foreach gobin,$(sort $(GO_TEST_WASM_BINS)),$(eval $(call gotestwasmfunc,$(gobin)))) +$(foreach gobin,$(sort $(GO_TEST_WASM_BINS)),$(eval FILE_IGNORES := $(FILE_IGNORES) $(gobin)/$(notdir $(gobin)))) diff --git a/private/buf/bufcli/cache.go b/private/buf/bufcli/cache.go index bd9a11f876..568e1104f0 100644 --- a/private/buf/bufcli/cache.go +++ b/private/buf/bufcli/cache.go @@ -103,6 +103,11 @@ var ( // // Normalized. v3CacheModuleLockRelDirPath = normalpath.Join("v3", "modulelocks") + // v3CacheWasmRuntimeRelDirPath is the relative path to the Wasm runtime cache directory in its newest iteration. + // This directory is used to store the Wasm runtime cache. This is an implementation specific cache and opaque outside of the runtime. + // + // Normalized. + v3CacheWasmRuntimeRelDirPath = normalpath.Join("v3", "wasmruntime") ) // NewModuleDataProvider returns a new ModuleDataProvider while creating the @@ -135,6 +140,19 @@ func NewCommitProvider(container appext.Container) (bufmodule.CommitProvider, er ) } +// CreateWasmRuntimeCacheDir creates the cache directory for the Wasm runtime. +// +// This is used by the Wasm runtime to cache compiled Wasm plugins. This is an +// implementation specific cache and opaque outside of the runtime. The runtime +// will manage the cache versioning itself within this directory. +func CreateWasmRuntimeCacheDir(container appext.Container) (string, error) { + if err := createCacheDir(container.CacheDirPath(), v3CacheWasmRuntimeRelDirPath); err != nil { + return "", err + } + fullCacheDirPath := normalpath.Join(container.CacheDirPath(), v3CacheWasmRuntimeRelDirPath) + return fullCacheDirPath, nil +} + // newWKTStore returns a new bufwktstore.Store while creating the required cache directories. func newWKTStore(container appext.Container) (bufwktstore.Store, error) { if err := createCacheDir(container.CacheDirPath(), v3CacheWKTRelDirPath); err != nil { diff --git a/private/buf/buflsp/buflsp.go b/private/buf/buflsp/buflsp.go index da572dfc11..06c3cc51e9 100644 --- a/private/buf/buflsp/buflsp.go +++ b/private/buf/buflsp/buflsp.go @@ -26,7 +26,6 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/pkg/app/appext" - "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/storage" "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/tracing" @@ -43,6 +42,7 @@ func Serve( ctx context.Context, container appext.Container, controller bufctl.Controller, + checkClient bufcheck.Client, stream jsonrpc2.Stream, ) (jsonrpc2.Conn, error) { // The LSP protocol deals with absolute filesystem paths. This requires us to @@ -57,12 +57,6 @@ func Serve( return nil, err } - tracer := tracing.NewTracer(container.Tracer()) - checkClient, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr())) - if err != nil { - return nil, err - } - conn := jsonrpc2.NewConn(stream) lsp := &lsp{ conn: conn, @@ -71,7 +65,7 @@ func Serve( zap.NewNop(), // The logging from protocol itself isn't very good, we've replaced it with connAdapter here. ), logger: container.Logger(), - tracer: tracer, + tracer: tracing.NewTracer(container.Tracer()), controller: controller, checkClient: checkClient, rootBucket: bucket, diff --git a/private/buf/bufmigrate/migrator.go b/private/buf/bufmigrate/migrator.go index 761f0787fd..a014ecd0ea 100644 --- a/private/buf/bufmigrate/migrator.go +++ b/private/buf/bufmigrate/migrator.go @@ -32,6 +32,7 @@ import ( "github.com/bufbuild/buf/private/pkg/storage" "github.com/bufbuild/buf/private/pkg/storage/storagemem" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/google/uuid" "go.uber.org/multierr" "go.uber.org/zap" @@ -712,7 +713,7 @@ func equivalentCheckConfigInV2( ) (bufconfig.CheckConfig, error) { // No need for custom lint/breaking plugins since there's no plugins to migrate from <=v1. // TODO: If we ever need v3, then we will have to deal with this. - client, err := bufcheck.NewClient(logger, tracer, bufcheck.NewRunnerProvider(runner)) + client, err := bufcheck.NewClient(logger, tracer, bufcheck.NewRunnerProvider(runner, wasm.UnimplementedRuntime)) if err != nil { return nil, err } diff --git a/private/buf/cmd/buf/buf_test.go b/private/buf/cmd/buf/buf_test.go index 4a3b0b7195..99949784c4 100644 --- a/private/buf/cmd/buf/buf_test.go +++ b/private/buf/cmd/buf/buf_test.go @@ -45,6 +45,7 @@ import ( "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/storage/storagetesting" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -1349,7 +1350,7 @@ func TestCheckLsBreakingRulesFromConfigExceptDeprecated(t *testing.T) { t.Run(version.String(), func(t *testing.T) { t.Parallel() // Do not need any custom lint/breaking plugins here. - client, err := bufcheck.NewClient(zap.NewNop(), tracing.NopTracer, bufcheck.NewRunnerProvider(command.NewRunner())) + client, err := bufcheck.NewClient(zap.NewNop(), tracing.NopTracer, bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime)) require.NoError(t, err) allRules, err := client.AllRules(context.Background(), check.RuleTypeBreaking, version) require.NoError(t, err) diff --git a/private/buf/cmd/buf/command/beta/lsp/lsp.go b/private/buf/cmd/buf/command/beta/lsp/lsp.go index e20dcf229d..7ff61d8c5e 100644 --- a/private/buf/cmd/buf/command/beta/lsp/lsp.go +++ b/private/buf/cmd/buf/command/beta/lsp/lsp.go @@ -25,11 +25,16 @@ import ( "github.com/bufbuild/buf/private/buf/bufcli" "github.com/bufbuild/buf/private/buf/buflsp" + "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" + "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/ioext" + "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/spf13/pflag" "go.lsp.dev/jsonrpc2" + "go.uber.org/multierr" ) const ( @@ -77,7 +82,7 @@ func run( ctx context.Context, container appext.Container, flags *flags, -) error { +) (retErr error) { bufcli.WarnBetaCommand(ctx, container) transport, err := dial(container, flags) @@ -90,7 +95,28 @@ func run( return err } - conn, err := buflsp.Serve(ctx, container, controller, jsonrpc2.NewStream(transport)) + wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container) + if err != nil { + return err + } + wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir)) + if err != nil { + return err + } + defer func() { + retErr = multierr.Append(retErr, wasmRuntime.Close(ctx)) + }() + checkClient, err := bufcheck.NewClient( + container.Logger(), + tracing.NewTracer(container.Tracer()), + bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime), + bufcheck.ClientWithStderr(container.Stderr()), + ) + if err != nil { + return err + } + + conn, err := buflsp.Serve(ctx, container, controller, checkClient, jsonrpc2.NewStream(transport)) if err != nil { return err } diff --git a/private/buf/cmd/buf/command/breaking/breaking.go b/private/buf/cmd/buf/command/breaking/breaking.go index 5b8263d20e..38e042005a 100644 --- a/private/buf/cmd/buf/command/breaking/breaking.go +++ b/private/buf/cmd/buf/command/breaking/breaking.go @@ -31,7 +31,9 @@ import ( "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/spf13/pflag" + "go.uber.org/multierr" ) const ( @@ -145,7 +147,7 @@ func run( ctx context.Context, container appext.Container, flags *flags, -) error { +) (retErr error) { if err := bufcli.ValidateRequiredFlag(againstFlagName, flags.Against); err != nil { return err } @@ -206,10 +208,26 @@ func run( len(againstImageWithConfigs), ) } + wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container) + if err != nil { + return err + } + wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir)) + if err != nil { + return err + } + defer func() { + retErr = multierr.Append(retErr, wasmRuntime.Close(ctx)) + }() tracer := tracing.NewTracer(container.Tracer()) var allFileAnnotations []bufanalysis.FileAnnotation for i, imageWithConfig := range imageWithConfigs { - client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr())) + client, err := bufcheck.NewClient( + container.Logger(), + tracer, + bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime), + bufcheck.ClientWithStderr(container.Stderr()), + ) if err != nil { return err } diff --git a/private/buf/cmd/buf/command/config/internal/internal.go b/private/buf/cmd/buf/command/config/internal/internal.go index 784f77717b..897440f3ac 100644 --- a/private/buf/cmd/buf/command/config/internal/internal.go +++ b/private/buf/cmd/buf/command/config/internal/internal.go @@ -32,7 +32,9 @@ import ( "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/bufbuild/buf/private/pkg/syserror" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/spf13/pflag" + "go.uber.org/multierr" ) const ( @@ -149,7 +151,7 @@ func lsRun( flags *flags, commandName string, ruleType check.RuleType, -) error { +) (retErr error) { if flags.ConfiguredOnly { if flags.Version != "" { return appcmd.NewInvalidArgumentErrorf("--%s cannot be specified if --%s is specified", versionFlagName, configFlagName) @@ -184,8 +186,24 @@ func lsRun( return err } } + wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container) + if err != nil { + return err + } + wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir)) + if err != nil { + return err + } + defer func() { + retErr = multierr.Append(retErr, wasmRuntime.Close(ctx)) + }() tracer := tracing.NewTracer(container.Tracer()) - client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr())) + client, err := bufcheck.NewClient( + container.Logger(), + tracer, + bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime), + bufcheck.ClientWithStderr(container.Stderr()), + ) if err != nil { return err } diff --git a/private/buf/cmd/buf/command/lint/lint.go b/private/buf/cmd/buf/command/lint/lint.go index 136b7b97dc..371b6e43de 100644 --- a/private/buf/cmd/buf/command/lint/lint.go +++ b/private/buf/cmd/buf/command/lint/lint.go @@ -28,7 +28,9 @@ import ( "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/spf13/pflag" + "go.uber.org/multierr" ) const ( @@ -131,10 +133,26 @@ func run( if err != nil { return err } + wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container) + if err != nil { + return err + } + wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir)) + if err != nil { + return err + } + defer func() { + retErr = multierr.Append(retErr, wasmRuntime.Close(ctx)) + }() tracer := tracing.NewTracer(container.Tracer()) var allFileAnnotations []bufanalysis.FileAnnotation for _, imageWithConfig := range imageWithConfigs { - client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr())) + client, err := bufcheck.NewClient( + container.Logger(), + tracer, + bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime), + bufcheck.ClientWithStderr(container.Stderr()), + ) if err != nil { return err } diff --git a/private/buf/cmd/buf/command/mod/internal/internal.go b/private/buf/cmd/buf/command/mod/internal/internal.go index 577a5c9e43..1247554d19 100644 --- a/private/buf/cmd/buf/command/mod/internal/internal.go +++ b/private/buf/cmd/buf/command/mod/internal/internal.go @@ -31,6 +31,7 @@ import ( "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/bufbuild/buf/private/pkg/syserror" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/spf13/pflag" ) @@ -175,7 +176,12 @@ func lsRun( } // BufYAMLFiles <=v1 never had plugins. tracer := tracing.NewTracer(container.Tracer()) - client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr())) + client, err := bufcheck.NewClient( + container.Logger(), + tracer, + bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime), + bufcheck.ClientWithStderr(container.Stderr()), + ) if err != nil { return err } diff --git a/private/buf/cmd/protoc-gen-buf-breaking/breaking.go b/private/buf/cmd/protoc-gen-buf-breaking/breaking.go index ec6f1d4bdf..522274a1a5 100644 --- a/private/buf/cmd/protoc-gen-buf-breaking/breaking.go +++ b/private/buf/cmd/protoc-gen-buf-breaking/breaking.go @@ -33,6 +33,7 @@ import ( "github.com/bufbuild/buf/private/pkg/protodescriptor" "github.com/bufbuild/buf/private/pkg/protoencoding" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/bufbuild/protoplugin" ) @@ -125,7 +126,12 @@ func handle( } // The protoc plugins do not support custom lint/breaking change plugins for now. tracer := tracing.NewTracer(container.Tracer()) - client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(pluginEnv.Stderr)) + client, err := bufcheck.NewClient( + container.Logger(), + tracer, + bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime), + bufcheck.ClientWithStderr(pluginEnv.Stderr), + ) if err != nil { return err } diff --git a/private/buf/cmd/protoc-gen-buf-lint/lint.go b/private/buf/cmd/protoc-gen-buf-lint/lint.go index 8f86b0ba47..b61a13edf2 100644 --- a/private/buf/cmd/protoc-gen-buf-lint/lint.go +++ b/private/buf/cmd/protoc-gen-buf-lint/lint.go @@ -32,6 +32,7 @@ import ( "github.com/bufbuild/buf/private/pkg/protodescriptor" "github.com/bufbuild/buf/private/pkg/protoencoding" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/bufbuild/protoplugin" ) @@ -100,7 +101,12 @@ func handle( } // The protoc plugins do not support custom lint/breaking change plugins for now. tracer := tracing.NewTracer(container.Tracer()) - client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(pluginEnv.Stderr)) + client, err := bufcheck.NewClient( + container.Logger(), + tracer, + bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime), + bufcheck.ClientWithStderr(pluginEnv.Stderr), + ) if err != nil { return err } diff --git a/private/bufpkg/bufcheck/breaking_test.go b/private/bufpkg/bufcheck/breaking_test.go index 53674249bd..3d53d63afa 100644 --- a/private/bufpkg/bufcheck/breaking_test.go +++ b/private/bufpkg/bufcheck/breaking_test.go @@ -33,6 +33,7 @@ import ( "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -1347,7 +1348,7 @@ func testBreaking( require.NoError(t, err) breakingConfig := workspace.GetBreakingConfigForOpaqueID(opaqueID) require.NotNil(t, breakingConfig) - client, err := bufcheck.NewClient(zap.NewNop(), tracing.NopTracer, bufcheck.NewRunnerProvider(command.NewRunner())) + client, err := bufcheck.NewClient(zap.NewNop(), tracing.NopTracer, bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime)) require.NoError(t, err) err = client.Breaking( ctx, diff --git a/private/bufpkg/bufcheck/bufcheck.go b/private/bufpkg/bufcheck/bufcheck.go index c921624fb3..7bfe732dde 100644 --- a/private/bufpkg/bufcheck/bufcheck.go +++ b/private/bufpkg/bufcheck/bufcheck.go @@ -22,10 +22,10 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/pkg/command" - "github.com/bufbuild/buf/private/pkg/pluginrpcutil" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/syserror" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "go.uber.org/zap" "pluginrpc.com/pluginrpc" ) @@ -165,26 +165,25 @@ type RunnerProvider interface { type RunnerProviderFunc func(pluginConfig bufconfig.PluginConfig) (pluginrpc.Runner, error) // NewRunner implements RunnerProvider. +// +// RunnerProvider selects the correct Runner based on the type of pluginConfig. func (r RunnerProviderFunc) NewRunner(pluginConfig bufconfig.PluginConfig) (pluginrpc.Runner, error) { return r(pluginConfig) } -// NewRunnerProvider returns a new RunnerProvider for the command.Runner. -func NewRunnerProvider(delegate command.Runner) RunnerProvider { - return RunnerProviderFunc( - func(pluginConfig bufconfig.PluginConfig) (pluginrpc.Runner, error) { - if pluginConfig.Type() != bufconfig.PluginConfigTypeLocal { - return nil, syserror.New("only local plugins are supported") - } - path := pluginConfig.Path() - return pluginrpcutil.NewRunner( - delegate, - // We know that Path is of at least length 1. - path[0], - path[1:]..., - ), nil - }, - ) +// NewRunnerProvider returns a new RunnerProvider for the command.Runner and wasm.Runtime. +// +// This implementation should only be used for local applications. It is safe to +// use concurrently. +// +// The RunnerProvider selects the correct Runner based on the PluginConfigType. +// The supported types are: +// - bufconfig.PluginConfigTypeLocal +// - bufconfig.PluginConfigTypeLocalWasm +// +// If the PluginConfigType is not supported, an error is returned. +func NewRunnerProvider(commandRunner command.Runner, wasmRuntime wasm.Runtime) RunnerProvider { + return newRunnerProvider(commandRunner, wasmRuntime) } // NewClient returns a new Client. diff --git a/private/bufpkg/bufcheck/client.go b/private/bufpkg/bufcheck/client.go index 02ac45852d..2753e01944 100644 --- a/private/bufpkg/bufcheck/client.go +++ b/private/bufpkg/bufcheck/client.go @@ -340,7 +340,7 @@ func (c *client) getMultiClient( newCheckClientSpec(pluginConfig.Name(), checkClient, options), ) } - return newMultiClient(c.logger, checkClientSpecs), nil + return newMultiClient(c.logger, c.tracer, checkClientSpecs), nil } func annotationsToFilteredFileAnnotationSetOrError( diff --git a/private/bufpkg/bufcheck/lint_test.go b/private/bufpkg/bufcheck/lint_test.go index 1f0e0f4c23..f554fa1a3a 100644 --- a/private/bufpkg/bufcheck/lint_test.go +++ b/private/bufpkg/bufcheck/lint_test.go @@ -30,6 +30,7 @@ import ( "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -1253,6 +1254,24 @@ func TestRunLintCustomPlugins(t *testing.T) { ) } +func TestRunLintCustomWasmPlugins(t *testing.T) { + t.Parallel() + if testing.Short() { + t.Skip("skipping test in short mode") + } + testLintWithOptions( + t, + "custom_wasm_plugins", + "", + nil, + bufanalysistesting.NewFileAnnotationNoLocation(t, "a.proto", "PACKAGE_DEFINED"), + bufanalysistesting.NewFileAnnotation(t, "a.proto", 8, 1, 10, 2, "SERVICE_BANNED_SUFFIXES"), + bufanalysistesting.NewFileAnnotation(t, "b.proto", 6, 3, 6, 66, "RPC_BANNED_SUFFIXES"), + bufanalysistesting.NewFileAnnotation(t, "b.proto", 14, 5, 14, 24, "ENUM_VALUE_BANNED_SUFFIXES"), + bufanalysistesting.NewFileAnnotation(t, "b.proto", 19, 5, 19, 23, "FIELD_BANNED_SUFFIXES"), + ) +} + func testLint( t *testing.T, relDirPath string, @@ -1275,7 +1294,7 @@ func testLintWithOptions( imageModifier func(bufimage.Image) bufimage.Image, expectedFileAnnotations ...bufanalysis.FileAnnotation, ) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) // Increased timeout for Wasm runtime defer cancel() baseDirPath := filepath.Join("testdata", "lint") @@ -1332,7 +1351,16 @@ func testLintWithOptions( lintConfig := workspace.GetLintConfigForOpaqueID(opaqueID) require.NotNil(t, lintConfig) - client, err := bufcheck.NewClient(zap.NewNop(), tracing.NopTracer, bufcheck.NewRunnerProvider(command.NewRunner())) + wasmRuntime, err := wasm.NewRuntime(ctx) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, wasmRuntime.Close(ctx)) + }) + client, err := bufcheck.NewClient( + zap.NewNop(), + tracing.NopTracer, + bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime), + ) require.NoError(t, err) err = client.Lint( ctx, diff --git a/private/bufpkg/bufcheck/multi_client.go b/private/bufpkg/bufcheck/multi_client.go index 861b55e4ce..0dbaba77b5 100644 --- a/private/bufpkg/bufcheck/multi_client.go +++ b/private/bufpkg/bufcheck/multi_client.go @@ -24,17 +24,21 @@ import ( "buf.build/go/bufplugin/check" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/thread" + "github.com/bufbuild/buf/private/pkg/tracing" + "go.opentelemetry.io/otel/attribute" "go.uber.org/zap" ) type multiClient struct { logger *zap.Logger + tracer tracing.Tracer checkClientSpecs []*checkClientSpec } -func newMultiClient(logger *zap.Logger, checkClientSpecs []*checkClientSpec) *multiClient { +func newMultiClient(logger *zap.Logger, tracer tracing.Tracer, checkClientSpecs []*checkClientSpec) *multiClient { return &multiClient{ logger: logger, + tracer: tracer, checkClientSpecs: checkClientSpecs, } } @@ -93,7 +97,15 @@ func (c *multiClient) Check(ctx context.Context, request check.Request) ([]*anno } jobs = append( jobs, - func(ctx context.Context) error { + func(ctx context.Context) (retErr error) { + ctx, span := c.tracer.Start( + ctx, + tracing.WithErr(&retErr), + tracing.WithAttributes( + attribute.Key("plugin").String(delegate.PluginName), + ), + ) + defer span.End() delegateResponse, err := delegate.Client.Check(ctx, delegateRequest) if err != nil { if delegate.PluginName == "" { @@ -148,6 +160,8 @@ func (c *multiClient) getRulesCategoriesAndChunkedIDs(ctx context.Context) ( retChunkedCategoryIDs [][]string, retErr error, ) { + ctx, span := c.tracer.Start(ctx, tracing.WithErr(&retErr)) + defer span.End() var rules []Rule chunkedRuleIDs := make([][]string, len(c.checkClientSpecs)) for i, delegate := range c.checkClientSpecs { diff --git a/private/bufpkg/bufcheck/multi_client_test.go b/private/bufpkg/bufcheck/multi_client_test.go index 2e0e0e464f..4561996031 100644 --- a/private/bufpkg/bufcheck/multi_client_test.go +++ b/private/bufpkg/bufcheck/multi_client_test.go @@ -28,6 +28,7 @@ import ( "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/bufbuild/buf/private/pkg/tracing" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -107,6 +108,7 @@ func testMultiClientSimple(t *testing.T, cacheRules bool) { require.NoError(t, err) multiClient := newMultiClient( zap.NewNop(), + tracing.NopTracer, []*checkClientSpec{ newCheckClientSpec("buf-plugin-field-lower-snake-case", fieldLowerSnakeCaseClient, emptyOptions), newCheckClientSpec("buf-plugin-timestamp-suffix", timestampSuffixClient, emptyOptions), @@ -167,6 +169,7 @@ func TestMultiClientCannotHaveOverlappingRules(t *testing.T) { require.NoError(t, err) multiClient := newMultiClient( zap.NewNop(), + tracing.NopTracer, []*checkClientSpec{ newCheckClientSpec("buf-plugin-field-lower-snake-case", fieldLowerSnakeCaseClient, emptyOptions), newCheckClientSpec("buf-plugin-field-lower-snake-case", fieldLowerSnakeCaseClient, emptyOptions), @@ -185,7 +188,7 @@ func TestMultiClientCannotHaveOverlappingRulesWithBuiltIn(t *testing.T) { client, err := newClient( zaptest.NewLogger(t), tracing.NopTracer, - NewRunnerProvider(command.NewRunner()), + NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime), ) require.NoError(t, err) duplicateBuiltInRulePluginConfig, err := bufconfig.NewLocalPluginConfig( @@ -261,6 +264,7 @@ func TestMultiClientCannotHaveOverlappingCategories(t *testing.T) { require.NoError(t, err) multiClient := newMultiClient( zap.NewNop(), + tracing.NopTracer, []*checkClientSpec{ newCheckClientSpec("buf-plugin-1", client1, emptyOptions), newCheckClientSpec("buf-plugin-2", client2, emptyOptions), @@ -279,7 +283,7 @@ func TestMultiClientCannotHaveOverlappingCategoriesWithBuiltIn(t *testing.T) { client, err := newClient( zaptest.NewLogger(t), tracing.NopTracer, - NewRunnerProvider(command.NewRunner()), + NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime), ) require.NoError(t, err) duplicateBuiltInRulePluginConfig, err := bufconfig.NewLocalPluginConfig( diff --git a/private/bufpkg/bufcheck/runner_provider.go b/private/bufpkg/bufcheck/runner_provider.go new file mode 100644 index 0000000000..e5788c9017 --- /dev/null +++ b/private/bufpkg/bufcheck/runner_provider.go @@ -0,0 +1,59 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufcheck + +import ( + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/pkg/command" + "github.com/bufbuild/buf/private/pkg/pluginrpcutil" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/wasm" + "pluginrpc.com/pluginrpc" +) + +type runnerProvider struct { + commandRunner command.Runner + wasmRuntime wasm.Runtime +} + +func newRunnerProvider(commandRunner command.Runner, wasmRuntime wasm.Runtime) *runnerProvider { + return &runnerProvider{ + commandRunner: commandRunner, + wasmRuntime: wasmRuntime, + } +} + +func (r *runnerProvider) NewRunner(pluginConfig bufconfig.PluginConfig) (pluginrpc.Runner, error) { + switch pluginConfig.Type() { + case bufconfig.PluginConfigTypeLocal: + path := pluginConfig.Path() + return pluginrpcutil.NewRunner( + r.commandRunner, + // We know that Path is of at least length 1. + path[0], + path[1:]..., + ), nil + case bufconfig.PluginConfigTypeLocalWasm: + path := pluginConfig.Path() + return pluginrpcutil.NewWasmRunner( + r.wasmRuntime, + // We know that Path is of at least length 1. + path[0], + path[1:]..., + ), nil + default: + return nil, syserror.Newf("unknown PluginConfigType: %v", pluginConfig.Type()) + } +} diff --git a/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/a.proto b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/a.proto new file mode 100644 index 0000000000..94d6ddf2a1 --- /dev/null +++ b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/a.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; + +service A { + rpc GetA(GetARequest) returns (GetAResponse); + rpc ListA(ListARequest) returns (ListAResponse); +} + +service AMock { + rpc GetAllA(GetAllARequest) returns (GetAllAResponse); +} + +message GetARequest {} +message GetAResponse {} + +message ListARequest { + uint32 page_size = 1; +} + +message ListAResponse { + message Value { + string id = 1; + bytes content = 2; + } + repeated Value values = 1; +} + +message GetAllARequest {} +message GetAllAResponse {} diff --git a/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/b.proto b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/b.proto new file mode 100644 index 0000000000..2dba34b847 --- /dev/null +++ b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/b.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package b; + +service B { + rpc GetElement(GetElementRequest) returns (GetElementResponse); +} + +message GetElementRequest {} +message GetElementResponse { + enum Status { + STATUS_UNSPECIFIED = 0; + STATUS_VALID = 1; + STATUS_INVALID = 2; + } + message Value { + string name = 1; + Status status = 2; + string a_uuid = 3; + } + Value value = 1; +} diff --git a/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/buf.yaml b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/buf.yaml new file mode 100644 index 0000000000..36f6c2273c --- /dev/null +++ b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/buf.yaml @@ -0,0 +1,20 @@ +version: v2 +lint: + use: + - PACKAGE_DEFINED + - SERVICE_BANNED_SUFFIXES + - RPC_BANNED_SUFFIXES + - FIELD_BANNED_SUFFIXES + - ENUM_VALUE_BANNED_SUFFIXES +plugins: + - plugin: buf-plugin-suffix.wasm + options: + service_banned_suffixes: + - Mock + - Test + rpc_banned_suffixes: + - Element + field_banned_suffixes: + - _uuid + enum_value_banned_suffixes: + - _INVALID diff --git a/private/bufpkg/bufconfig/plugin_config.go b/private/bufpkg/bufconfig/plugin_config.go index f3e8543b7c..388bb15b50 100644 --- a/private/bufpkg/bufconfig/plugin_config.go +++ b/private/bufpkg/bufconfig/plugin_config.go @@ -16,6 +16,8 @@ package bufconfig import ( "errors" + "fmt" + "path/filepath" "strings" "github.com/bufbuild/buf/private/pkg/encoding" @@ -25,6 +27,8 @@ import ( const ( // PluginConfigTypeLocal is the local plugin config type. PluginConfigTypeLocal PluginConfigType = iota + 1 + // PluginConfigTypeLocalWasm is the local Wasm plugin config type. + PluginConfigTypeLocalWasm ) // PluginConfigType is a generate plugin configuration type. @@ -62,6 +66,23 @@ func NewLocalPluginConfig( ) } +// NewLocalWasmPluginConfig returns a new PluginConfig for a local Wasm plugin. +// +// The first path argument is the path to the Wasm plugin and must end with .wasm. +// The remaining path arguments are the arguments to the Wasm plugin. These are passed +// to the Wasm plugin as command line arguments. +func NewLocalWasmPluginConfig( + name string, + options map[string]any, + path []string, +) (PluginConfig, error) { + return newLocalWasmPluginConfig( + name, + options, + path, + ) +} + // *** PRIVATE *** type pluginConfig struct { @@ -91,6 +112,17 @@ func newPluginConfigForExternalV2( if err != nil { return nil, err } + if len(path) == 0 { + return nil, errors.New("must specify a path to the plugin") + } + // Wasm plugins are suffixed with .wasm. Otherwise, it's a binary. + if filepath.Ext(path[0]) == ".wasm" { + return newLocalWasmPluginConfig( + strings.Join(path, " "), + options, + path, + ) + } return newLocalPluginConfig( strings.Join(path, " "), options, @@ -114,6 +146,25 @@ func newLocalPluginConfig( }, nil } +func newLocalWasmPluginConfig( + name string, + options map[string]any, + path []string, +) (*pluginConfig, error) { + if len(path) == 0 { + return nil, errors.New("must specify a path to the plugin") + } + if filepath.Ext(path[0]) != ".wasm" { + return nil, fmt.Errorf("must specify a path to the plugin, and the first path argument must end with .wasm") + } + return &pluginConfig{ + pluginConfigType: PluginConfigTypeLocalWasm, + name: name, + options: options, + path: path, + }, nil +} + func (p *pluginConfig) Type() PluginConfigType { return p.pluginConfigType } diff --git a/private/pkg/pluginrpcutil/pluginrpcutil.go b/private/pkg/pluginrpcutil/pluginrpcutil.go index f984f8b8a9..ef50a20146 100644 --- a/private/pkg/pluginrpcutil/pluginrpcutil.go +++ b/private/pkg/pluginrpcutil/pluginrpcutil.go @@ -16,6 +16,7 @@ package pluginrpcutil import ( "github.com/bufbuild/buf/private/pkg/command" + "github.com/bufbuild/buf/private/pkg/wasm" "pluginrpc.com/pluginrpc" ) @@ -23,3 +24,10 @@ import ( func NewRunner(delegate command.Runner, programName string, programArgs ...string) pluginrpc.Runner { return newRunner(delegate, programName, programArgs...) } + +// NewWasmRunner returns a new pluginrpc.Runner for the wasm.Runtime and program name. +// +// This runner is used for local Wasm plugins. The program name is the path to the Wasm file. +func NewWasmRunner(delegate wasm.Runtime, programName string, programArgs ...string) pluginrpc.Runner { + return newWasmRunner(delegate, programName, programArgs...) +} diff --git a/private/pkg/pluginrpcutil/wasm_runner.go b/private/pkg/pluginrpcutil/wasm_runner.go new file mode 100644 index 0000000000..0c7640ab41 --- /dev/null +++ b/private/pkg/pluginrpcutil/wasm_runner.go @@ -0,0 +1,120 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pluginrpcutil + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "slices" + "sync" + + "github.com/bufbuild/buf/private/pkg/wasm" + "pluginrpc.com/pluginrpc" +) + +type wasmRunner struct { + delegate wasm.Runtime + programName string + programArgs []string + // lock protects compiledModule and compiledModuleErr. Store called as + // a boolean to avoid nil comparison. + lock sync.RWMutex + called bool + compiledModule wasm.CompiledModule + compiledModuleErr error +} + +func newWasmRunner( + delegate wasm.Runtime, + programName string, + programArgs ...string, +) *wasmRunner { + return &wasmRunner{ + delegate: delegate, + programName: programName, + programArgs: programArgs, + } +} + +func (r *wasmRunner) Run(ctx context.Context, env pluginrpc.Env) (retErr error) { + compiledModule, err := r.loadCompiledModuleOnce(ctx) + if err != nil { + return err + } + if len(r.programArgs) > 0 { + env.Args = append(slices.Clone(r.programArgs), env.Args...) + } + return compiledModule.Run(ctx, env) +} + +func (r *wasmRunner) loadCompiledModuleOnce(ctx context.Context) (wasm.CompiledModule, error) { + r.lock.RLock() + if r.called { + r.lock.RUnlock() + return r.compiledModule, r.compiledModuleErr + } + r.lock.RUnlock() + r.lock.Lock() + defer r.lock.Unlock() + if !r.called { + r.compiledModule, r.compiledModuleErr = r.loadCompiledModule(ctx) + r.called = true + } + return r.compiledModule, r.compiledModuleErr +} + +func (r *wasmRunner) loadCompiledModule(ctx context.Context) (wasm.CompiledModule, error) { + // Find the plugin path. We use the same logic as exec.LookPath, but we do + // not require the file to be executable. So check the local directory + // first before checking the PATH. + var path string + if fileInfo, err := os.Stat(r.programName); err == nil && !fileInfo.IsDir() { + path = r.programName + } else { + var err error + path, err = unsafeLookPath(r.programName) + if err != nil { + return nil, fmt.Errorf("could not find plugin %q in PATH: %v", r.programName, err) + } + } + moduleWasm, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("could not read plugin %q: %v", r.programName, err) + } + // Compile the module. This CompiledModule is never released, so + // subsequent calls to this function will benefit from the cached + // module. This is only safe as the runner is limited to the CLI. + compiledModule, err := r.delegate.Compile(ctx, r.programName, moduleWasm) + if err != nil { + return nil, err + } + return compiledModule, nil +} + +// unsafeLookPath is a wrapper around exec.LookPath that restores the original +// pre-Go 1.19 behavior of resolving queries that would use relative PATH +// entries. We consider it acceptable for the use case of locating plugins. +// +// https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory +func unsafeLookPath(file string) (string, error) { + path, err := exec.LookPath(file) + if errors.Is(err, exec.ErrDot) { + err = nil + } + return path, err +} diff --git a/private/pkg/wasm/compiled_module.go b/private/pkg/wasm/compiled_module.go new file mode 100644 index 0000000000..84766013df --- /dev/null +++ b/private/pkg/wasm/compiled_module.go @@ -0,0 +1,65 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "context" + "fmt" + + "github.com/tetratelabs/wazero" + "pluginrpc.com/pluginrpc" +) + +type compiledModule struct { + moduleName string + runtime wazero.Runtime + compiledModule wazero.CompiledModule +} + +func (p *compiledModule) Run(ctx context.Context, env pluginrpc.Env) error { + // Create a new module wazeroModuleConfig with the given environment. + wazeroModuleConfig := wazero.NewModuleConfig(). + WithStdin(env.Stdin). + WithStdout(env.Stdout). + WithStderr(env.Stderr). + // Use an empty name to allow for multiple instances of the same module. + // See wazero.ModuleConfig.WithName. + WithName(""). + // Use the program name as the first argument to replicate the + // behavior of os.Exec. + // See wazero.ModuleConfig.WithArgs. + WithArgs(append([]string{p.moduleName}, env.Args...)...) + + // Instantiate the Wasm module into the runtime. This effectively runs + // the Wasm module. Only the effect of instantiating the module is used, + // the module is closed immediately after running to free up resources. + // See https://github.com/tetratelabs/wazero/issues/985. + wazeroModule, err := p.runtime.InstantiateModule( + ctx, + p.compiledModule, + wazeroModuleConfig, + ) + if err != nil { + return fmt.Errorf("failed to instantiate module: %w", err) + } + if err := wazeroModule.Close(ctx); err != nil { + return fmt.Errorf("failed to close module: %w", err) + } + return nil +} + +func (p *compiledModule) Close(ctx context.Context) error { + return p.compiledModule.Close(ctx) +} diff --git a/private/pkg/wasm/runtime.go b/private/pkg/wasm/runtime.go new file mode 100644 index 0000000000..d80178847f --- /dev/null +++ b/private/pkg/wasm/runtime.go @@ -0,0 +1,131 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "context" + "fmt" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" + "go.uber.org/multierr" +) + +const ( + // defaultMaxMemoryBytes is the maximum memory size in bytes. + defaultMaxMemoryBytes = 1 << 29 // 512 MiB + // wasmPageSize is the page size in bytes. + wasmPageSize = 1 << 16 // 64 KiB +) + +type runtime struct { + runtime wazero.Runtime + compilationCache wazero.CompilationCache +} + +func newRuntime(ctx context.Context, options ...RuntimeOption) (*runtime, error) { + runtimeOptions := newRuntimeOptions() + for _, option := range options { + option(runtimeOptions) + } + if runtimeOptions.maxMemoryBytes == 0 { + return nil, fmt.Errorf("Wasm max memory bytes must be greater than 0") + } + // The maximum memory size is limited to 4 GiB. Sizes less than the page + // size (64 KiB) are truncated. memoryLimitPages is guaranteed to be + // below 2^16 as the maxium uint32 value is 2^32 - 1. + // NOTE: The option represented as a uint32 restricts the max number of + // pages to 2^16-1, one less the the actual max value of 2^16. But this + // is a nicer API then specifying the max number of pages directly. + memoryLimitPages := runtimeOptions.maxMemoryBytes / wasmPageSize + if memoryLimitPages == 0 { + return nil, fmt.Errorf("Wasm max memory bytes %d is too small", runtimeOptions.maxMemoryBytes) + } + + // Create the wazero.RuntimeConfig with enforceable limits. Limits are + // enforced through the Wasm sandbox. The following limits are set: + // - Memory limit: The maximum memory size in pages. + // - CPU limit: The runtime stops work on context done. + // - Access limit: All system interfaces are stubbed. No network, + // disk, clock, etc. + // See wazero.NewRuntimeConfig for more details. + wazeroRuntimeConfig := wazero.NewRuntimeConfig(). + WithCoreFeatures(api.CoreFeaturesV2). + WithCloseOnContextDone(true). + WithMemoryLimitPages(memoryLimitPages) + var wazeroCompilationCache wazero.CompilationCache + if runtimeOptions.cacheDir != "" { + var err error + wazeroCompilationCache, err = wazero.NewCompilationCacheWithDir(runtimeOptions.cacheDir) + if err != nil { + return nil, fmt.Errorf("failed to create compilation cache: %w", err) + } + wazeroRuntimeConfig = wazeroRuntimeConfig.WithCompilationCache(wazeroCompilationCache) + } + wazeroRuntime := wazero.NewRuntimeWithConfig(ctx, wazeroRuntimeConfig) + + // Init WASI preview1 APIs. This is required to support the pluginrpc + // protocol. The returned closer method is discarded as the + // instantiated module is never required to be unloaded. + if _, err := wasi_snapshot_preview1.Instantiate(ctx, wazeroRuntime); err != nil { + return nil, fmt.Errorf("failed to instantiate WASI snapshot preview1: %w", err) + } + return &runtime{ + runtime: wazeroRuntime, + compilationCache: wazeroCompilationCache, + }, nil +} + +func (r *runtime) Compile(ctx context.Context, moduleName string, moduleWasm []byte) (CompiledModule, error) { + if moduleName == "" { + // The plugin is required to be named. We cannot use the name + // from the Wasm binary as this is not guaranteed to be set and + // may conflict with the provided name. + return nil, fmt.Errorf("name is empty") + } + // Compile the WebAssembly. This operation is hashed on the module + // bytes and the runtime configuration. The compiled module is + // cached in memory and on disk if an optional cache directory is + // provided. + wazeroCompiledModule, err := r.runtime.CompileModule(ctx, moduleWasm) + if err != nil { + return nil, err + } + return &compiledModule{ + moduleName: moduleName, + runtime: r.runtime, + compiledModule: wazeroCompiledModule, + }, nil +} + +func (r *runtime) Close(ctx context.Context) error { + err := r.runtime.Close(ctx) + if r.compilationCache != nil { + err = multierr.Append(err, r.compilationCache.Close(ctx)) + } + return err +} + +type runtimeOptions struct { + maxMemoryBytes uint32 + cacheDir string +} + +func newRuntimeOptions() *runtimeOptions { + return &runtimeOptions{ + maxMemoryBytes: defaultMaxMemoryBytes, + } +} diff --git a/private/pkg/wasm/unimplemented_runtime.go b/private/pkg/wasm/unimplemented_runtime.go new file mode 100644 index 0000000000..14be4975f2 --- /dev/null +++ b/private/pkg/wasm/unimplemented_runtime.go @@ -0,0 +1,31 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "context" + + "github.com/bufbuild/buf/private/pkg/syserror" +) + +type unimplementedRuntime struct{} + +func (unimplementedRuntime) Compile(ctx context.Context, name string, moduleBytes []byte) (CompiledModule, error) { + return nil, syserror.Newf("not implemented") +} + +func (unimplementedRuntime) Close(ctx context.Context) error { + return nil +} diff --git a/private/pkg/wasm/usage.gen.go b/private/pkg/wasm/usage.gen.go new file mode 100644 index 0000000000..8fbec91f11 --- /dev/null +++ b/private/pkg/wasm/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package wasm + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/pkg/wasm/wasm.go b/private/pkg/wasm/wasm.go new file mode 100644 index 0000000000..5275a0a432 --- /dev/null +++ b/private/pkg/wasm/wasm.go @@ -0,0 +1,81 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package wasm provides a Wasm runtime for plugins. +package wasm + +import ( + "context" + + "pluginrpc.com/pluginrpc" +) + +// UnimplementedRuntime is an unimplemented Runtime. +var UnimplementedRuntime = unimplementedRuntime{} + +// CompiledModule is a Wasm module ready to be run. +// +// It is safe to use this module concurrently. When done, call Close to free +// resources held by the CompiledModule. All CompiledModules created by the +// same Runtime will be invalidated when the Runtime is closed. +// +// Memory is limited by the Runtime. To restrict CPU usage, cancel the context. +type CompiledModule interface { + pluginrpc.Runner + // Close releases all resources held by the compiled module. + Close(ctx context.Context) error +} + +// Runtime is a Wasm runtime. +// +// It is safe to use the Runtime concurrently. Close must be called when done +// with the Runtime to free resources. All CompiledModules created by the same +// Runtime will be invalidated when the Runtime is closed. +type Runtime interface { + // Compile compiles the given Wasm module bytes into a CompiledModule. + Compile(ctx context.Context, moduleName string, moduleWasm []byte) (CompiledModule, error) + // Close releases all resources held by the Runtime. + Close(ctx context.Context) error +} + +// NewRuntime creates a new Wasm Runtime. +func NewRuntime(ctx context.Context, options ...RuntimeOption) (Runtime, error) { + return newRuntime(ctx, options...) +} + +// RuntimeOption is an option for Runtime. +type RuntimeOption func(*runtimeOptions) + +// WithMaxMemoryBytes sets the maximum memory size in bytes. +// +// The maximuim memory size is limited to 4 GiB. The default is 512 MiB. Sizes +// less then the page size (64 KiB) are truncated. +func WithMaxMemoryBytes(maxMemoryBytes uint32) RuntimeOption { + return func(runtimeOptions *runtimeOptions) { + runtimeOptions.maxMemoryBytes = maxMemoryBytes + } +} + +// WithLocalCacheDir sets the local cache directory. +// +// The cache directory is used to store compiled Wasm modules. This can be used +// to speed up subsequent runs of the same module. The internal cache structure +// and versioning is handled by the Runtime. +// +// This option is only safe use in CLI environments. +func WithLocalCacheDir(cacheDir string) RuntimeOption { + return func(runtimeOptions *runtimeOptions) { + runtimeOptions.cacheDir = cacheDir + } +} diff --git a/private/usage/usage_unix.go b/private/usage/usage_unix.go index 55ffab50f2..5c3fd8d29a 100644 --- a/private/usage/usage_unix.go +++ b/private/usage/usage_unix.go @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build aix || darwin || dragonfly || freebsd || (js && wasm) || linux || netbsd || openbsd || solaris -// +build aix darwin dragonfly freebsd js,wasm linux netbsd openbsd solaris +//go:build unix || wasip1 || js package usage diff --git a/private/usage/usage_windows.go b/private/usage/usage_windows.go index 05697bd5c3..e52aa9252f 100644 --- a/private/usage/usage_windows.go +++ b/private/usage/usage_windows.go @@ -13,7 +13,6 @@ // limitations under the License. //go:build windows -// +build windows package usage diff --git a/windows/test.bash b/windows/test.bash index 242d6edb54..522f81f5b0 100644 --- a/windows/test.bash +++ b/windows/test.bash @@ -40,4 +40,6 @@ go install ./cmd/buf \ ./private/bufpkg/bufcheck/internal/cmd/buf-plugin-rpc-ext \ ./private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-category \ ./private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-rule +GOOS=wasip1 GOARCH=wasm go build -o $(go env -json | jq -r .GOPATH)/bin/buf-plugin-suffix.wasm \ + ./private/bufpkg/bufcheck/internal/cmd/buf-plugin-suffix go test ./...