From 6cd717840e63fe114a1b470cffd8a54c306d06aa Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Mon, 23 Sep 2024 16:25:30 -0400 Subject: [PATCH] Init support for WASM plugins --- go.mod | 1 + go.sum | 2 + make/buf/all.mk | 7 + make/go/go.mk | 18 +- private/buf/bufcli/cache.go | 15 ++ private/buf/buflsp/buflsp.go | 7 +- .../buf/cmd/buf/command/breaking/breaking.go | 7 +- private/buf/cmd/buf/command/lint/lint.go | 24 ++- private/bufpkg/bufcheck/bufcheck.go | 35 ++-- private/bufpkg/bufcheck/lint_test.go | 50 ++++- private/bufpkg/bufcheck/multi_client.go | 7 + private/bufpkg/bufcheck/runner_provider.go | 184 ++++++++++++++++++ .../testdata/lint/custom_wasm_plugins/a.proto | 28 +++ .../testdata/lint/custom_wasm_plugins/b.proto | 22 +++ .../lint/custom_wasm_plugins/buf.yaml | 20 ++ private/bufpkg/bufconfig/plugin_config.go | 48 ++++- private/bufpkg/bufwasm/bufwasm.go | 81 ++++++++ private/bufpkg/bufwasm/plugin.go | 66 +++++++ private/bufpkg/bufwasm/runtime.go | 129 ++++++++++++ private/bufpkg/bufwasm/usage.gen.go | 19 ++ private/usage/usage_unix.go | 3 +- private/usage/usage_windows.go | 1 - 22 files changed, 743 insertions(+), 31 deletions(-) create mode 100644 private/bufpkg/bufcheck/runner_provider.go create mode 100644 private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/a.proto create mode 100644 private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/b.proto create mode 100644 private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/buf.yaml create mode 100644 private/bufpkg/bufwasm/bufwasm.go create mode 100644 private/bufpkg/bufwasm/plugin.go create mode 100644 private/bufpkg/bufwasm/runtime.go create mode 100644 private/bufpkg/bufwasm/usage.gen.go diff --git a/go.mod b/go.mod index 76117b6431..9e0d7be733 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 + github.com/tetratelabs/wazero v1.8.0 go.lsp.dev/jsonrpc2 v0.10.0 go.lsp.dev/protocol v0.12.0 go.opentelemetry.io/otel v1.30.0 diff --git a/go.sum b/go.sum index c49d5bf29d..437a89e4f8 100644 --- a/go.sum +++ b/go.sum @@ -252,6 +252,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g= +github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/make/buf/all.mk b/make/buf/all.mk index 4f0aa08257..84512e4bf7 100644 --- a/make/buf/all.mk +++ b/make/buf/all.mk @@ -25,6 +25,13 @@ GO_TEST_BINS := $(GO_TEST_BINS) \ private/bufpkg/bufcheck/internal/cmd/buf-plugin-rpc-ext \ private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-category \ private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-rule +GO_TEST_WASM_BINS := $(GO_TEST_WASM_BINS) \ + private/bufpkg/bufcheck/internal/cmd/buf-plugin-panic \ + private/bufpkg/bufcheck/internal/cmd/buf-plugin-suffix \ + private/bufpkg/bufcheck/internal/cmd/buf-plugin-protovalidate-ext \ + private/bufpkg/bufcheck/internal/cmd/buf-plugin-rpc-ext \ + private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-category \ + private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-rule GO_MOD_VERSION := 1.22 DOCKER_BINS := $(DOCKER_BINS) buf FILE_IGNORES := $(FILE_IGNORES) \ diff --git a/make/go/go.mk b/make/go/go.mk index 503d72a53b..36b356efcb 100644 --- a/make/go/go.mk +++ b/make/go/go.mk @@ -16,6 +16,8 @@ GO_BINS ?= # Settable GO_TEST_BINS ?= # Settable +GO_TEST_WASM_BINS ?= +# Settable GO_GET_PKGS ?= # Settable GO_MOD_VERSION ?= 1.21 @@ -142,7 +144,7 @@ build: prebuild ## Run go build. pretest:: .PHONY: test -test: pretest installtest ## Run all go tests. +test: pretest installtest installtestwasm ## Run all go tests. go test $(GO_TEST_FLAGS) $(GOPKGS) .PHONY: testrace @@ -203,3 +205,17 @@ endef $(foreach gobin,$(sort $(GO_TEST_BINS)),$(eval $(call gotestbinfunc,$(gobin)))) $(foreach gobin,$(sort $(GO_TEST_BINS)),$(eval FILE_IGNORES := $(FILE_IGNORES) $(gobin)/$(notdir $(gobin)))) + +.PHONY: installtestwasm +installtestwasm:: + +define gotestwasmfunc +.PHONY: installtestwasm$(notdir $(1)) +installtestwasm$(notdir $(1)): + GOOS=wasip1 GOARCH=wasm go build -o $(GOBIN)/$(notdir $(1)).wasm ./$(1) + +installtestwasm:: installtestwasm$(notdir $(1)) +endef + +$(foreach gobin,$(sort $(GO_TEST_WASM_BINS)),$(eval $(call gotestwasmfunc,$(gobin)))) +$(foreach gobin,$(sort $(GO_TEST_WASM_BINS)),$(eval FILE_IGNORES := $(FILE_IGNORES) $(gobin)/$(notdir $(gobin)))) diff --git a/private/buf/bufcli/cache.go b/private/buf/bufcli/cache.go index bd9a11f876..17f1900424 100644 --- a/private/buf/bufcli/cache.go +++ b/private/buf/bufcli/cache.go @@ -103,6 +103,10 @@ var ( // // Normalized. v3CacheModuleLockRelDirPath = normalpath.Join("v3", "modulelocks") + // v3CachePluginsRelDirPath is the relative path to the plugins cache directory in its newest iteration. + // + // Normalized. + v3CachePluginsRelDirPath = normalpath.Join("v3", "plugins") ) // NewModuleDataProvider returns a new ModuleDataProvider while creating the @@ -135,6 +139,17 @@ func NewCommitProvider(container appext.Container) (bufmodule.CommitProvider, er ) } +// CreatePluginCacheDir creates the cache directory for plugins. +// +// This is used by the [bufwasm.WithLocalCacheDir] option. +func CreatePluginCacheDir(container appext.Container) (string, error) { + if err := createCacheDir(container.CacheDirPath(), v3CachePluginsRelDirPath); err != nil { + return "", err + } + fullCacheDirPath := normalpath.Join(container.CacheDirPath(), v3CachePluginsRelDirPath) + return fullCacheDirPath, nil +} + // newWKTStore returns a new bufwktstore.Store while creating the required cache directories. func newWKTStore(container appext.Container) (bufwktstore.Store, error) { if err := createCacheDir(container.CacheDirPath(), v3CacheWKTRelDirPath); err != nil { diff --git a/private/buf/buflsp/buflsp.go b/private/buf/buflsp/buflsp.go index da572dfc11..5865d91bc4 100644 --- a/private/buf/buflsp/buflsp.go +++ b/private/buf/buflsp/buflsp.go @@ -58,7 +58,12 @@ func Serve( } tracer := tracing.NewTracer(container.Tracer()) - checkClient, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr())) + checkClient, err := bufcheck.NewClient( + container.Logger(), + tracer, + bufcheck.NewRunnerProvider(command.NewRunner()), + bufcheck.ClientWithStderr(container.Stderr()), + ) if err != nil { return nil, err } diff --git a/private/buf/cmd/buf/command/breaking/breaking.go b/private/buf/cmd/buf/command/breaking/breaking.go index 5b8263d20e..007e860b77 100644 --- a/private/buf/cmd/buf/command/breaking/breaking.go +++ b/private/buf/cmd/buf/command/breaking/breaking.go @@ -209,7 +209,12 @@ func run( tracer := tracing.NewTracer(container.Tracer()) var allFileAnnotations []bufanalysis.FileAnnotation for i, imageWithConfig := range imageWithConfigs { - client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr())) + client, err := bufcheck.NewClient( + container.Logger(), + tracer, + bufcheck.NewRunnerProvider(command.NewRunner()), + bufcheck.ClientWithStderr(container.Stderr()), + ) if err != nil { return err } diff --git a/private/buf/cmd/buf/command/lint/lint.go b/private/buf/cmd/buf/command/lint/lint.go index 136b7b97dc..534c1c9b57 100644 --- a/private/buf/cmd/buf/command/lint/lint.go +++ b/private/buf/cmd/buf/command/lint/lint.go @@ -23,12 +23,14 @@ import ( "github.com/bufbuild/buf/private/buf/bufctl" "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufcheck" + "github.com/bufbuild/buf/private/bufpkg/bufwasm" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/bufbuild/buf/private/pkg/tracing" "github.com/spf13/pflag" + "go.uber.org/multierr" ) const ( @@ -131,10 +133,30 @@ func run( if err != nil { return err } + pluginCacheDir, err := bufcli.CreatePluginCacheDir(container) + if err != nil { + return err + } + wasmRuntime, err := bufwasm.NewRuntime( + ctx, + bufwasm.WithLocalCacheDir(pluginCacheDir), + ) + if err != nil { + return err + } + defer func() { retErr = multierr.Append(retErr, wasmRuntime.Release(ctx)) }() tracer := tracing.NewTracer(container.Tracer()) var allFileAnnotations []bufanalysis.FileAnnotation for _, imageWithConfig := range imageWithConfigs { - client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr())) + client, err := bufcheck.NewClient( + container.Logger(), + tracer, + bufcheck.NewRunnerProvider( + command.NewRunner(), + bufcheck.RunnerProviderWithWASMRuntime(wasmRuntime), + ), + bufcheck.ClientWithStderr(container.Stderr()), + ) if err != nil { return err } diff --git a/private/bufpkg/bufcheck/bufcheck.go b/private/bufpkg/bufcheck/bufcheck.go index c921624fb3..403fb0ed54 100644 --- a/private/bufpkg/bufcheck/bufcheck.go +++ b/private/bufpkg/bufcheck/bufcheck.go @@ -21,8 +21,8 @@ import ( "buf.build/go/bufplugin/check" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufwasm" "github.com/bufbuild/buf/private/pkg/command" - "github.com/bufbuild/buf/private/pkg/pluginrpcutil" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/syserror" "github.com/bufbuild/buf/private/pkg/tracing" @@ -170,21 +170,24 @@ func (r RunnerProviderFunc) NewRunner(pluginConfig bufconfig.PluginConfig) (plug } // NewRunnerProvider returns a new RunnerProvider for the command.Runner. -func NewRunnerProvider(delegate command.Runner) RunnerProvider { - return RunnerProviderFunc( - func(pluginConfig bufconfig.PluginConfig) (pluginrpc.Runner, error) { - if pluginConfig.Type() != bufconfig.PluginConfigTypeLocal { - return nil, syserror.New("only local plugins are supported") - } - path := pluginConfig.Path() - return pluginrpcutil.NewRunner( - delegate, - // We know that Path is of at least length 1. - path[0], - path[1:]..., - ), nil - }, - ) +// +// This implementation should only be used for local applications. +func NewRunnerProvider( + delegate command.Runner, + options ...RunnerProviderOption, +) RunnerProvider { + return newRunnerProvider(delegate, options...) +} + +// RunnerProviderOption is an option for NewRunnerProvider. +type RunnerProviderOption func(*runnerProviderOptions) + +// RunnerProviderWithWASMRuntime returns a new RunnerProviderOption that +// specifies a WASM runtime. This is required for local WASM plugins. +func RunnerProviderWithWASMRuntime(wasmRuntime bufwasm.Runtime) RunnerProviderOption { + return func(runnerProviderOptions *runnerProviderOptions) { + runnerProviderOptions.wasmRuntime = wasmRuntime + } } // NewClient returns a new Client. diff --git a/private/bufpkg/bufcheck/lint_test.go b/private/bufpkg/bufcheck/lint_test.go index 4e93324b0d..610fafe406 100644 --- a/private/bufpkg/bufcheck/lint_test.go +++ b/private/bufpkg/bufcheck/lint_test.go @@ -27,6 +27,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufwasm" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/tracing" @@ -511,6 +512,7 @@ func TestRunPackageNoImportCycle(t *testing.T) { require.NoError(t, err) return newImage }, + false, bufanalysistesting.NewFileAnnotation(t, "c1.proto", 5, 1, 5, 19, "PACKAGE_NO_IMPORT_CYCLE"), bufanalysistesting.NewFileAnnotation(t, "d1.proto", 5, 1, 5, 19, "PACKAGE_NO_IMPORT_CYCLE"), ) @@ -593,7 +595,8 @@ func TestRunProtovalidate(t *testing.T) { t, "protovalidate", "buf.testing/lint/protovalidate", - nil, + nil, // no image modification + false, // no wasm runtime bufanalysistesting.NewFileAnnotation(t, "bool.proto", 18, 51, 18, 84, "PROTOVALIDATE"), bufanalysistesting.NewFileAnnotation(t, "bool.proto", 19, 31, 19, 69, "PROTOVALIDATE"), bufanalysistesting.NewFileAnnotation(t, "bool.proto", 20, 50, 20, 88, "PROTOVALIDATE"), @@ -1026,7 +1029,8 @@ func TestRunV2WorkspaceIgnores(t *testing.T) { t, "v2/ignores", "ignores1", - nil, + nil, // no image modification + false, // no wasm runtime bufanalysistesting.NewFileAnnotation(t, "bar1/bar.proto", 6, 9, 6, 15, "FIELD_LOWER_SNAKE_CASE"), bufanalysistesting.NewFileAnnotation(t, "bar1/bar.proto", 9, 9, 9, 12, "MESSAGE_PASCAL_CASE"), bufanalysistesting.NewFileAnnotation(t, "bar1/bar.proto", 13, 6, 13, 9, "ENUM_PASCAL_CASE"), @@ -1050,7 +1054,8 @@ func TestRunV2WorkspaceIgnores(t *testing.T) { t, "v2/ignores", "ignores2", - nil, + nil, // no image modification + false, // no wasm runtime bufanalysistesting.NewFileAnnotation(t, "bar2/bar.proto", 6, 9, 6, 15, "FIELD_LOWER_SNAKE_CASE"), bufanalysistesting.NewFileAnnotation(t, "bar2/bar.proto", 9, 9, 9, 12, "MESSAGE_PASCAL_CASE"), bufanalysistesting.NewFileAnnotation(t, "bar2/bar.proto", 13, 6, 13, 9, "ENUM_PASCAL_CASE"), @@ -1062,7 +1067,8 @@ func TestRunV2WorkspaceIgnores(t *testing.T) { t, "v2/ignores", "ignores3", - nil, + nil, // no image modification + false, // no wasm runtime bufanalysistesting.NewFileAnnotation(t, "bar3/bar.proto", 6, 9, 6, 15, "FIELD_LOWER_SNAKE_CASE"), bufanalysistesting.NewFileAnnotation(t, "bar3/bar2.proto", 6, 9, 6, 15, "FIELD_LOWER_SNAKE_CASE"), bufanalysistesting.NewFileAnnotation(t, "bar3/bar2.proto", 9, 9, 9, 13, "MESSAGE_PASCAL_CASE"), @@ -1214,6 +1220,25 @@ func TestRunLintCustomPlugins(t *testing.T) { ) } +func TestRunLintCustomWASMPlugins(t *testing.T) { + t.Parallel() + if testing.Short() { + t.Skip("skipping test in short mode") + } + testLintWithOptions( + t, + "custom_wasm_plugins", + "", + nil, // no image modification + true, // wasm runtime + bufanalysistesting.NewFileAnnotationNoLocation(t, "a.proto", "PACKAGE_DEFINED"), + bufanalysistesting.NewFileAnnotation(t, "a.proto", 8, 1, 10, 2, "SERVICE_BANNED_SUFFIXES"), + bufanalysistesting.NewFileAnnotation(t, "b.proto", 6, 3, 6, 66, "RPC_BANNED_SUFFIXES"), + bufanalysistesting.NewFileAnnotation(t, "b.proto", 14, 5, 14, 24, "ENUM_VALUE_BANNED_SUFFIXES"), + bufanalysistesting.NewFileAnnotation(t, "b.proto", 19, 5, 19, 23, "FIELD_BANNED_SUFFIXES"), + ) +} + func testLint( t *testing.T, relDirPath string, @@ -1224,6 +1249,7 @@ func testLint( relDirPath, "", nil, + false, expectedFileAnnotations..., ) } @@ -1234,9 +1260,10 @@ func testLintWithOptions( // only set if in workspace moduleFullNameString string, imageModifier func(bufimage.Image) bufimage.Image, + wasmRuntime bool, expectedFileAnnotations ...bufanalysis.FileAnnotation, ) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) // Increased timeout for WASM runtime defer cancel() baseDirPath := filepath.Join("testdata", "lint") @@ -1293,7 +1320,18 @@ func testLintWithOptions( lintConfig := workspace.GetLintConfigForOpaqueID(opaqueID) require.NotNil(t, lintConfig) - client, err := bufcheck.NewClient(zap.NewNop(), tracing.NopTracer, bufcheck.NewRunnerProvider(command.NewRunner())) + var runnerProviderOptions []bufcheck.RunnerProviderOption + if wasmRuntime { + wasmRuntime, err := bufwasm.NewRuntime(ctx) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, wasmRuntime.Release(ctx)) }) + runnerProviderOptions = append(runnerProviderOptions, bufcheck.RunnerProviderWithWASMRuntime(wasmRuntime)) + } + client, err := bufcheck.NewClient( + zap.NewNop(), + tracing.NopTracer, + bufcheck.NewRunnerProvider(command.NewRunner(), runnerProviderOptions...), + ) require.NoError(t, err) err = client.Lint( ctx, diff --git a/private/bufpkg/bufcheck/multi_client.go b/private/bufpkg/bufcheck/multi_client.go index 861b55e4ce..83b6cdaa4d 100644 --- a/private/bufpkg/bufcheck/multi_client.go +++ b/private/bufpkg/bufcheck/multi_client.go @@ -20,6 +20,7 @@ import ( "sort" "strings" "sync" + "time" "buf.build/go/bufplugin/check" "github.com/bufbuild/buf/private/pkg/slicesext" @@ -94,6 +95,7 @@ func (c *multiClient) Check(ctx context.Context, request check.Request) ([]*anno jobs = append( jobs, func(ctx context.Context) error { + start := time.Now() delegateResponse, err := delegate.Client.Check(ctx, delegateRequest) if err != nil { if delegate.PluginName == "" { @@ -110,6 +112,7 @@ func (c *multiClient) Check(ctx context.Context, request check.Request) ([]*anno lock.Lock() allAnnotations = append(allAnnotations, annotations...) lock.Unlock() + c.logger.Debug("checked delegate client", zap.String("pluginName", delegate.PluginName), zap.Duration("duration", time.Since(start))) return nil }, ) @@ -151,6 +154,7 @@ func (c *multiClient) getRulesCategoriesAndChunkedIDs(ctx context.Context) ( var rules []Rule chunkedRuleIDs := make([][]string, len(c.checkClientSpecs)) for i, delegate := range c.checkClientSpecs { + start := time.Now() delegateCheckRules, err := delegate.Client.ListRules(ctx) if err != nil { if delegate.PluginName == "" { @@ -165,11 +169,13 @@ func (c *multiClient) getRulesCategoriesAndChunkedIDs(ctx context.Context) ( rules = append(rules, delegateRules...) // Already sorted. chunkedRuleIDs[i] = slicesext.Map(delegateRules, Rule.ID) + c.logger.Debug("list rules delegate client", zap.String("pluginName", delegate.PluginName), zap.Duration("duration", time.Since(start))) } var categories []Category chunkedCategoryIDs := make([][]string, len(c.checkClientSpecs)) for i, delegate := range c.checkClientSpecs { + start := time.Now() delegateCheckCategories, err := delegate.Client.ListCategories(ctx) if err != nil { if delegate.PluginName == "" { @@ -184,6 +190,7 @@ func (c *multiClient) getRulesCategoriesAndChunkedIDs(ctx context.Context) ( categories = append(categories, delegateCategories...) // Already sorted. chunkedCategoryIDs[i] = slicesext.Map(delegateCategories, Category.ID) + c.logger.Debug("list categories delegate client", zap.String("pluginName", delegate.PluginName), zap.Duration("duration", time.Since(start))) } if err := validateNoDuplicateRulesOrCategories(rules, categories); err != nil { diff --git a/private/bufpkg/bufcheck/runner_provider.go b/private/bufpkg/bufcheck/runner_provider.go new file mode 100644 index 0000000000..7279711432 --- /dev/null +++ b/private/bufpkg/bufcheck/runner_provider.go @@ -0,0 +1,184 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufcheck + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufwasm" + "github.com/bufbuild/buf/private/pkg/command" + "github.com/bufbuild/buf/private/pkg/pluginrpcutil" + "github.com/bufbuild/buf/private/pkg/syserror" + "pluginrpc.com/pluginrpc" +) + +type runnerProvider struct { + delegate command.Runner + wasmRuntime bufwasm.Runtime +} + +func newRunnerProvider(delegate command.Runner, options ...RunnerProviderOption) *runnerProvider { + runnerProviderOptions := newRunnerProviderOptions() + for _, option := range options { + option(runnerProviderOptions) + } + return &runnerProvider{ + delegate: delegate, + wasmRuntime: runnerProviderOptions.wasmRuntime, + } +} + +func (r *runnerProvider) NewRunner(pluginConfig bufconfig.PluginConfig) (pluginrpc.Runner, error) { + switch pluginConfig.Type() { + case bufconfig.PluginConfigTypeLocal: + path := pluginConfig.Path() + return pluginrpcutil.NewRunner( + r.delegate, + // We know that Path is of at least length 1. + path[0], + path[1:]..., + ), nil + case bufconfig.PluginConfigTypeLocalWASM: + if r.wasmRuntime == nil { + return nil, syserror.Newf("wasm runtime is required for local wasm plugins") + } + return newWASMRunner( + r.wasmRuntime, + pluginConfig.Name(), + ), nil + default: + return nil, syserror.Newf("unsupported plugin type: %v", pluginConfig.Type()) + } +} + +type runnerProviderOptions struct { + wasmRuntime bufwasm.Runtime +} + +func newRunnerProviderOptions() *runnerProviderOptions { + return &runnerProviderOptions{ + wasmRuntime: bufwasm.NewUnimplementedRuntime(), + } +} + +// wasmRunner is a runner that loads a WASM plugin. +type wasmRunner struct { + programName string + wasmRuntime bufwasm.Runtime + // Once protects plugin and pluginErr. + once sync.Once + plugin bufwasm.Plugin + pluginErr error +} + +// newWASMRunner returns a new pluginrpc.Runner for the WASM binary on a +// bufwasm.Runtime and program name. This runner is only suitable for use with +// short-lived programs, compiled plugins lifetime is tied to the runtime. +// +// The program name should be the name of the program as it appears in the +// plugin config. The runner will call os.GetEnv("PATH") with os.Stat on each +// directory and file to find the program. This is similar to exec.LookPath +// but does not require the file to be executable. This is only safe for use +// in the CLI, as it is not safe to use in a server environment. +func newWASMRunner( + runtime bufwasm.Runtime, + programName string, +) *wasmRunner { + return &wasmRunner{ + programName: programName, + wasmRuntime: runtime, + } +} + +func (r *wasmRunner) Run(ctx context.Context, env pluginrpc.Env) (retErr error) { + plugin, err := r.loadPluginOnce(ctx) + if err != nil { + return err + } + return plugin.Run(ctx, env) +} + +func (r *wasmRunner) loadPluginOnce(ctx context.Context) (bufwasm.Plugin, error) { + r.once.Do(func() { + r.plugin, r.pluginErr = r.loadPlugin(ctx) + }) + return r.plugin, r.pluginErr +} + +func (r *wasmRunner) loadPlugin(ctx context.Context) (bufwasm.Plugin, error) { + path, err := lookPath(r.programName) + if err != nil { + return nil, fmt.Errorf("could not find plugin %q in PATH: %v", r.programName, err) + } + wasmModule, err := os.ReadFile(path) + if err != nil { + return nil, err + } + // Compile and run, releasing the plugin at the end. + plugin, err := r.wasmRuntime.Compile(ctx, r.programName, wasmModule) + if err != nil { + return nil, err + } + // This plugin is never released, so subsequent calls to this function + // will benefit from the cached plugin. This is only safe as the + // runner is limited to the CLI. + return plugin, nil +} + +// lookPath looks for a wasm file in the PATH, not only an executable. This +// doesn't use exec.LookPath to avoid requiring an executable bit. +func lookPath(file string) (string, error) { + // First, check in the current directory. + if ok, err := findFile(file); err != nil { + return "", err + } else if ok { + return file, nil + } + // If the file has a path separator, fail early. + if strings.Contains(file, string(os.PathSeparator)) { + return "", os.ErrNotExist + } + path := os.Getenv("PATH") + for _, dir := range filepath.SplitList(path) { + if dir == "" { + // Unix shell semantics: path element "" means "." + dir = "." + } + path := filepath.Join(dir, file) + if ok, err := findFile(path); err != nil { + return "", err + } else if ok { + return path, nil + } + } + return "", os.ErrNotExist +} + +func findFile(file string) (bool, error) { + d, err := os.Stat(file) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return !d.Mode().IsDir(), nil +} diff --git a/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/a.proto b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/a.proto new file mode 100644 index 0000000000..94d6ddf2a1 --- /dev/null +++ b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/a.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; + +service A { + rpc GetA(GetARequest) returns (GetAResponse); + rpc ListA(ListARequest) returns (ListAResponse); +} + +service AMock { + rpc GetAllA(GetAllARequest) returns (GetAllAResponse); +} + +message GetARequest {} +message GetAResponse {} + +message ListARequest { + uint32 page_size = 1; +} + +message ListAResponse { + message Value { + string id = 1; + bytes content = 2; + } + repeated Value values = 1; +} + +message GetAllARequest {} +message GetAllAResponse {} diff --git a/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/b.proto b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/b.proto new file mode 100644 index 0000000000..2dba34b847 --- /dev/null +++ b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/b.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package b; + +service B { + rpc GetElement(GetElementRequest) returns (GetElementResponse); +} + +message GetElementRequest {} +message GetElementResponse { + enum Status { + STATUS_UNSPECIFIED = 0; + STATUS_VALID = 1; + STATUS_INVALID = 2; + } + message Value { + string name = 1; + Status status = 2; + string a_uuid = 3; + } + Value value = 1; +} diff --git a/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/buf.yaml b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/buf.yaml new file mode 100644 index 0000000000..36f6c2273c --- /dev/null +++ b/private/bufpkg/bufcheck/testdata/lint/custom_wasm_plugins/buf.yaml @@ -0,0 +1,20 @@ +version: v2 +lint: + use: + - PACKAGE_DEFINED + - SERVICE_BANNED_SUFFIXES + - RPC_BANNED_SUFFIXES + - FIELD_BANNED_SUFFIXES + - ENUM_VALUE_BANNED_SUFFIXES +plugins: + - plugin: buf-plugin-suffix.wasm + options: + service_banned_suffixes: + - Mock + - Test + rpc_banned_suffixes: + - Element + field_banned_suffixes: + - _uuid + enum_value_banned_suffixes: + - _INVALID diff --git a/private/bufpkg/bufconfig/plugin_config.go b/private/bufpkg/bufconfig/plugin_config.go index f3e8543b7c..ab94813798 100644 --- a/private/bufpkg/bufconfig/plugin_config.go +++ b/private/bufpkg/bufconfig/plugin_config.go @@ -18,6 +18,7 @@ import ( "errors" "strings" + "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/pkg/encoding" "github.com/bufbuild/buf/private/pkg/syserror" ) @@ -25,6 +26,8 @@ import ( const ( // PluginConfigTypeLocal is the local plugin config type. PluginConfigTypeLocal PluginConfigType = iota + 1 + // PluginConfigTypeLocalWASM is the local WASM plugin config type. + PluginConfigTypeLocalWASM ) // PluginConfigType is a generate plugin configuration type. @@ -62,6 +65,18 @@ func NewLocalPluginConfig( ) } +// NewLocalWASMPluginConfig returns a new PluginConfig for a local WASM plugin. +func NewLocalWASMPluginConfig( + name string, + options map[string]any, + path []string, +) (PluginConfig, error) { + return newLocalWASMPluginConfig( + name, + options, + ) +} + // *** PRIVATE *** type pluginConfig struct { @@ -85,12 +100,30 @@ func newPluginConfigForExternalV2( } options[key] = value } - // TODO: differentiate between local and remote in the future - // Use the same heuristic that we do for dir vs module in buffetch + // Differentiate between local and remote plugins. + // Local plugins are specified as a path to a binary or a WASM file. + // Remote plugins are specified as a module reference. + // Paths with more than one element are assumed to be local plugin commands. + // This heuristic is based on the buffetch heuristics for inputs. path, err := encoding.InterfaceSliceOrStringToStringSlice(externalConfig.Plugin) if err != nil { return nil, err } + if len(path) == 1 { + name := path[0] + // TODO: Parse as a module reference to fail early if we find + // a remote plugin reference. This syntax is subject to change. + if _, err := bufmodule.ParseModuleRef(name); err == nil { + return nil, syserror.Newf("remote plugins are not yet supported") + } + // WASM plugins are suffixed with .wasm. Otherwise, it's a binary. + if strings.HasSuffix(name, ".wasm") { + return newLocalWASMPluginConfig( + name, + options, + ) + } + } return newLocalPluginConfig( strings.Join(path, " "), options, @@ -114,6 +147,17 @@ func newLocalPluginConfig( }, nil } +func newLocalWASMPluginConfig( + name string, + options map[string]any, +) (*pluginConfig, error) { + return &pluginConfig{ + pluginConfigType: PluginConfigTypeLocalWASM, + name: name, + options: options, + }, nil +} + func (p *pluginConfig) Type() PluginConfigType { return p.pluginConfigType } diff --git a/private/bufpkg/bufwasm/bufwasm.go b/private/bufpkg/bufwasm/bufwasm.go new file mode 100644 index 0000000000..0aa2302efd --- /dev/null +++ b/private/bufpkg/bufwasm/bufwasm.go @@ -0,0 +1,81 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bufwasm provides a WASM runtime for plugins. +package bufwasm + +import ( + "context" + + "pluginrpc.com/pluginrpc" +) + +// Plugin is a WASM module. +// +// It is safe to use this plugin concurrently. Ensure that you call [Release] +// when you are done with the plugin. +// +// Memory is limited by the runtime. To restrict CPU usage, cancel the context. +type Plugin interface { + pluginrpc.Runner + // Name returns the name of the plugin. + Name() string + // Release releases all resources held by the plugin. + Release(ctx context.Context) error +} + +// Runtime is a WASM runtime. +// +// It is safe to use this runtime concurrently. Ensure that you call [Release] +// when you are done with the runtime. All plugins created by this runtime will +// be invalidated when [Release] is called. +type Runtime interface { + // Compile compiles the given module into a [Plugin]. + // + // The plugin is not validated to conform to the pluginrpc protocol. + Compile(ctx context.Context, pluginName string, pluginWASM []byte) (Plugin, error) + // Release releases all resources held by the runtime. + Release(ctx context.Context) error +} + +// NewRuntime creates a new WASM runtime. +func NewRuntime(ctx context.Context, options ...RuntimeOption) (Runtime, error) { + return newRuntime(ctx, options...) +} + +// RuntimeOption is an option for [NewRuntime]. +type RuntimeOption interface { + apply(*runtimeConfig) +} + +// WithMaxMemoryBytes sets the maximum memory size in bytes. +func WithMaxMemoryBytes(maxMemoryBytes uint32) RuntimeOption { + return runtimeOptionFunc(func(cfg *runtimeConfig) { + cfg.maxMemoryBytes = maxMemoryBytes + }) +} + +// WithLocalCacheDir sets the local cache directory. +// +// This option is only safe use in CLI environments. +func WithLocalCacheDir(cacheDir string) RuntimeOption { + return runtimeOptionFunc(func(cfg *runtimeConfig) { + cfg.cacheDir = cacheDir + }) +} + +// NewUnimplementedRuntime returns a new unimplemented Runtime. +func NewUnimplementedRuntime() Runtime { + return unimplementedRuntime{} +} diff --git a/private/bufpkg/bufwasm/plugin.go b/private/bufpkg/bufwasm/plugin.go new file mode 100644 index 0000000000..d289a22ff0 --- /dev/null +++ b/private/bufpkg/bufwasm/plugin.go @@ -0,0 +1,66 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufwasm + +import ( + "context" + "fmt" + + "github.com/tetratelabs/wazero" + "pluginrpc.com/pluginrpc" +) + +type plugin struct { + name string + runtime wazero.Runtime + compiledModule wazero.CompiledModule +} + +var _ Plugin = (*plugin)(nil) + +func (p *plugin) Name() string { + return p.name +} + +func (p *plugin) Run(ctx context.Context, env pluginrpc.Env) error { + // Create a new module config with the given environment. + config := wazero.NewModuleConfig(). + WithStdin(env.Stdin). + WithStdout(env.Stdout). + WithStderr(env.Stderr) + + // Instantiate the guest wasm module into the same runtime. + // See: https://github.com/tetratelabs/wazero/issues/985 + mod, err := p.runtime.InstantiateModule( + ctx, + p.compiledModule, + // Use an empty name to allow for multiple instances of the same module. + // See [wazero.ModuleConfig.WithName]. + config.WithName("").WithArgs( + append([]string{p.name}, env.Args...)..., + ), + ) + if err != nil { + return fmt.Errorf("failed to instantiate module: %w", err) + } + if err := mod.Close(ctx); err != nil { + return fmt.Errorf("failed to close module: %w", err) + } + return nil +} + +func (p *plugin) Release(ctx context.Context) error { + return p.compiledModule.Close(ctx) +} diff --git a/private/bufpkg/bufwasm/runtime.go b/private/bufpkg/bufwasm/runtime.go new file mode 100644 index 0000000000..223190a6db --- /dev/null +++ b/private/bufpkg/bufwasm/runtime.go @@ -0,0 +1,129 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufwasm + +import ( + "context" + "fmt" + + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" + "go.uber.org/multierr" +) + +const ( + // defaultMaxMemoryBytes is the maximum memory size in bytes. + defaultMaxMemoryBytes = 1 << 29 // 512 MiB + // wasmPageSize is the page size in bytes. + wasmPageSize = 1 << 16 // 64 KiB +) + +type runtime struct { + runtime wazero.Runtime + cache wazero.CompilationCache +} + +var _ Runtime = (*runtime)(nil) + +func newRuntime(ctx context.Context, options ...RuntimeOption) (*runtime, error) { + var cfg runtimeConfig + for _, opt := range options { + opt.apply(&cfg) + } + // Create the runtime config with enforceable limits. + runtimeConfig := wazero.NewRuntimeConfig(). + WithCoreFeatures(api.CoreFeaturesV2). + WithCloseOnContextDone(true). + WithMemoryLimitPages(cfg.getMaxMemoryBytes() / wasmPageSize) + var cache wazero.CompilationCache + if cfg.cacheDir != "" { + var err error + cache, err = wazero.NewCompilationCacheWithDir(cfg.cacheDir) + if err != nil { + return nil, fmt.Errorf("failed to create compilation cache: %w", err) + } + runtimeConfig = runtimeConfig.WithCompilationCache(cache) + } + r := wazero.NewRuntimeWithConfig(ctx, runtimeConfig) + + // Init wasi. + wasi_snapshot_preview1.MustInstantiate(ctx, r) + + return &runtime{ + runtime: r, + cache: cache, + }, nil +} + +func (r *runtime) Compile(ctx context.Context, name string, module []byte) (Plugin, error) { + if name == "" { + // The plugin is required to be named. We cannot use the name + // from the WASM binary as this is not guaranteed to be set and + // may conflict with the provided name. + return nil, fmt.Errorf("name is empty") + } + // Compile the module. This operation is hashed on module by the wazero + // runtime. + compiledModule, err := r.runtime.CompileModule(ctx, module) + if err != nil { + return nil, err + } + return &plugin{ + name: name, + runtime: r.runtime, + compiledModule: compiledModule, + }, nil +} + +func (r *runtime) Release(ctx context.Context) error { + err := r.runtime.Close(ctx) + if r.cache != nil { + err = multierr.Append(err, r.cache.Close(ctx)) + } + return err +} + +type runtimeConfig struct { + maxMemoryBytes uint32 + cacheDir string +} + +func (r *runtimeConfig) getMaxMemoryBytes() uint32 { + if r.maxMemoryBytes == 0 { + return defaultMaxMemoryBytes + } + return r.maxMemoryBytes +} + +type runtimeOptionFunc func(*runtimeConfig) + +func (f runtimeOptionFunc) apply(cfg *runtimeConfig) { + f(cfg) +} + +var _ RuntimeOption = runtimeOptionFunc(nil) + +type unimplementedRuntime struct{} + +var _ Runtime = unimplementedRuntime{} + +func (unimplementedRuntime) Compile(ctx context.Context, name string, module []byte) (Plugin, error) { + return nil, syserror.Newf("not implemented") +} +func (unimplementedRuntime) Release(ctx context.Context) error { + return nil +} diff --git a/private/bufpkg/bufwasm/usage.gen.go b/private/bufpkg/bufwasm/usage.gen.go new file mode 100644 index 0000000000..b0c8ba81b7 --- /dev/null +++ b/private/bufpkg/bufwasm/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufwasm + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/usage/usage_unix.go b/private/usage/usage_unix.go index 55ffab50f2..5c3fd8d29a 100644 --- a/private/usage/usage_unix.go +++ b/private/usage/usage_unix.go @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build aix || darwin || dragonfly || freebsd || (js && wasm) || linux || netbsd || openbsd || solaris -// +build aix darwin dragonfly freebsd js,wasm linux netbsd openbsd solaris +//go:build unix || wasip1 || js package usage diff --git a/private/usage/usage_windows.go b/private/usage/usage_windows.go index 05697bd5c3..e52aa9252f 100644 --- a/private/usage/usage_windows.go +++ b/private/usage/usage_windows.go @@ -13,7 +13,6 @@ // limitations under the License. //go:build windows -// +build windows package usage