Enable fetching plugins from github repository

changelog/unreleased/fetch-github-plugins.md

@@ -0,0 +1,5 @@
+Enhancement: Enable fetching plugins from github
+
+This PR introduces new feature of fetching plugin from github
+
+https://github.com/cs3org/reva/pull/1970

examples/plugin/json/json.go

@@ -27,6 +27,7 @@ import (
 
 	userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
 	"github.com/cs3org/reva/pkg/errtypes"
+	revaPlugin "github.com/cs3org/reva/pkg/plugin"
 	"github.com/cs3org/reva/pkg/user"
 	"github.com/hashicorp/go-plugin"
 	"github.com/mitchellh/mapstructure"
@@ -145,16 +146,9 @@ func (m *Manager) GetUserGroups(ctx context.Context, uid *userpb.UserId) ([]stri
 	return user.Groups, nil
 }
 
-// Handshake hashicorp go-plugin handshake
-var Handshake = plugin.HandshakeConfig{
-	ProtocolVersion:  1,
-	MagicCookieKey:   "BASIC_PLUGIN",
-	MagicCookieValue: "hello",
-}
-
 func main() {
 	plugin.Serve(&plugin.ServeConfig{
-		HandshakeConfig: Handshake,
+		HandshakeConfig: revaPlugin.Handshake,
 		Plugins: map[string]plugin.Plugin{
 			"userprovider": &user.ProviderPlugin{Impl: &Manager{}},
 		},

examples/plugin/plugin.toml

@@ -9,7 +9,13 @@ address = "0.0.0.0:19000"
 userprovidersvc = "localhost:19000"
 
 [grpc.services.userprovider]
-driver = "./json"
+driver = "/home/jimil/Desktop/reva/examples/plugin/json"
 
 [grpc.services.userprovider.drivers.json]
 users = "users.demo.json"
+
+[grpc.services.authprovider]
+auth_manager = "github.com/cs3org/reva-auth-json-plugin"
+
+[grpc.services.autprovider.auth_managers.reva-auth-json-plugin]
+users = "users.demo.json"

go.mod

@@ -2,7 +2,6 @@ module github.com/cs3org/reva
 
 require (
 	bou.ke/monkey v1.0.2
-	contrib.go.opencensus.io/exporter/jaeger v0.2.1
 	contrib.go.opencensus.io/exporter/prometheus v0.3.0
 	github.com/BurntSushi/toml v0.4.1
 	github.com/Masterminds/goutils v1.1.0 // indirect
@@ -33,7 +32,8 @@ require (
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/uuid v1.3.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
-	github.com/hashicorp/go-hclog v0.16.2
+	github.com/hashicorp/go-getter v1.5.6
+	github.com/hashicorp/go-hclog v0.14.1
 	github.com/hashicorp/go-plugin v1.4.2
 	github.com/huandu/xstrings v1.3.0 // indirect
 	github.com/imdario/mergo v0.3.8 // indirect
@@ -61,13 +61,13 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.22.0
 	go.opentelemetry.io/otel v1.0.0-RC2
 	go.opentelemetry.io/otel/exporters/jaeger v1.0.0-RC2
-	go.opentelemetry.io/otel/metric v0.20.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.0.0-RC2
 	go.opentelemetry.io/otel/trace v1.0.0-RC2
 	golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1
+	google.golang.org/api v0.29.0 // indirect
 	google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 // indirect
 	google.golang.org/grpc v1.39.1
 	google.golang.org/protobuf v1.27.1
@@ -81,5 +81,4 @@ go 1.16
 replace (
 	github.com/eventials/go-tus => github.com/andrewmostello/go-tus v0.0.0-20200314041820-904a9904af9a
 	github.com/oleiade/reflections => github.com/oleiade/reflections v1.0.1
-	google.golang.org/grpc => google.golang.org/grpc v1.26.0 // temporary downgrade
 )

pkg/auth/rpc_auth.go

@@ -23,8 +23,7 @@ import (
 	"net/rpc"
 
 	authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
-	user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
-	"github.com/cs3org/reva/pkg/appctx"
+	userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
 	"github.com/cs3org/reva/pkg/plugin"
 	hcplugin "github.com/hashicorp/go-plugin"
 )
@@ -74,24 +73,27 @@ func (m *RPCClient) Configure(ml map[string]interface{}) error {
 
 // AuthenticateArgs for RPC
 type AuthenticateArgs struct {
-	Ctx          map[interface{}]interface{}
+	Ctx          *plugin.Ctx
 	ClientID     string
 	ClientSecret string
 }
 
 // AuthenticateReply for RPC
 type AuthenticateReply struct {
-	User  *user.User
+	User  *userpb.User
 	Auth  map[string]*authpb.Scope
 	Error error
 }
 
 // Authenticate RPCClient Authenticate method
-func (m *RPCClient) Authenticate(ctx context.Context, clientID, clientSecret string) (*user.User, map[string]*authpb.Scope, error) {
-	ctxVal := appctx.GetKeyValuesFromCtx(ctx)
+func (m *RPCClient) Authenticate(ctx context.Context, clientID, clientSecret string) (*userpb.User, map[string]*authpb.Scope, error) {
+	ctxVal, err := plugin.GetContextStruct(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
 	args := AuthenticateArgs{Ctx: ctxVal, ClientID: clientID, ClientSecret: clientSecret}
 	reply := AuthenticateReply{}
-	err := m.Client.Call("Plugin.Authenticate", args, &reply)
+	err = m.Client.Call("Plugin.Authenticate", args, &reply)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -112,7 +114,7 @@ func (m *RPCServer) Configure(args ConfigureArg, resp *ConfigureReply) error {
 
 // Authenticate RPCServer Authenticate method
 func (m *RPCServer) Authenticate(args AuthenticateArgs, resp *AuthenticateReply) error {
-	ctx := appctx.PutKeyValuesToCtx(args.Ctx)
+	ctx := plugin.SetContext(args.Ctx)
 	resp.User, resp.Auth, resp.Error = m.Impl.Authenticate(ctx, args.ClientID, args.ClientSecret)
 	return nil
 }

pkg/plugin/context.go

@@ -0,0 +1,56 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package plugin
+
+import (
+	"context"
+	"fmt"
+
+	userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
+	ctxpkg "github.com/cs3org/reva/pkg/ctx"
+)
+
+// Ctx represents context to be passed to the plugins
+type Ctx struct {
+	User  *userpb.User
+	Token string
+}
+
+// GetContextStruct retrieves context KV pairs and stores it into Ctx
+func GetContextStruct(ctx context.Context) (*Ctx, error) {
+	var ok bool
+	ctxVal := &Ctx{}
+	ctxVal.User, ok = ctxpkg.ContextGetUser(ctx)
+	if !ok {
+		return nil, fmt.Errorf("cannot get user context")
+	}
+	ctxVal.Token, ok = ctxpkg.ContextGetToken(ctx)
+	if !ok {
+		return nil, fmt.Errorf("cannot get token context")
+	}
+	return ctxVal, nil
+}
+
+// SetContext sets the context
+func SetContext(ctxStruct *Ctx) context.Context {
+	ctx := context.Background()
+	ctx = ctxpkg.ContextSetUser(ctx, ctxStruct.User)
+	ctx = ctxpkg.ContextSetToken(ctx, ctxStruct.Token)
+	return ctx
+}

pkg/plugin/loader.go

@@ -20,13 +20,15 @@ package plugin
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
 
 	"github.com/cs3org/reva/pkg/errtypes"
+	"github.com/hashicorp/go-getter"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-plugin"
 )
@@ -39,24 +41,27 @@ type RevaPlugin struct {
 
 const dirname = "/var/tmp/reva"
 
-var isAlphaNum = regexp.MustCompile(`^[A-Za-z0-9]+$`).MatchString
+var isAlphaNum = regexp.MustCompile(`^[A-Za-z0-9_-]+$`).MatchString
+var isURL = regexp.MustCompile(`^(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$`).MatchString
 
 // Kill kills the plugin process
 func (plug *RevaPlugin) Kill() {
 	plug.Client.Kill()
 }
 
-var handshake = plugin.HandshakeConfig{
+// Handshake contains plugin Handshake configuration
+var Handshake = plugin.HandshakeConfig{
 	ProtocolVersion:  1,
 	MagicCookieKey:   "BASIC_PLUGIN",
-	MagicCookieValue: "hello",
+	MagicCookieValue: "reva",
 }
 
 func compile(pluginType string, path string) (string, error) {
 	var errb bytes.Buffer
 	binaryPath := filepath.Join(dirname, "bin", pluginType, filepath.Base(path))
 	command := fmt.Sprintf("go build -o %s %s", binaryPath, path)
 	cmd := exec.Command("bash", "-c", command)
+	cmd.Dir = path
 	cmd.Stderr = &errb
 	err := cmd.Run()
 	if err != nil {
@@ -65,7 +70,7 @@ func compile(pluginType string, path string) (string, error) {
 	return binaryPath, nil
 }
 
-// checkDir checks and compiles plugin if the configuration points to a directory.
+// checkDirAndCompile checks and compiles plugin if the configuration points to a directory.
 func checkDirAndCompile(pluginType, driver string) (string, error) {
 	bin := driver
 	file, err := os.Stat(driver)
@@ -82,24 +87,65 @@ func checkDirAndCompile(pluginType, driver string) (string, error) {
 	return bin, nil
 }
 
-// Load loads the plugin using the hashicorp go-plugin system
-func Load(pluginType, driver string) (*RevaPlugin, error) {
+// downloadPlugin downloads the plugin and stores it into local filesystem
+func downloadAndCompilePlugin(pluginType, driver string) (string, error) {
+	destination := fmt.Sprintf("%s/ext/%s/%s", dirname, pluginType, filepath.Base(driver))
+	client := &getter.Client{
+		Ctx:  context.Background(),
+		Dst:  destination,
+		Src:  driver,
+		Mode: getter.ClientModeDir,
+	}
+	if err := client.Get(); err != nil {
+		return "", err
+	}
+	bin, err := compile(pluginType, destination)
+	if err != nil {
+		return "", nil
+	}
+	return bin, nil
+}
+
+// isValidURL tests a string to determine if it is a well-structure URL
+func isValidURL(driver string) bool {
+	return isURL(driver)
+}
+
+func fetchBinary(pluginType, driver string) (string, error) {
+	var bin string
+	var err error
 	if isAlphaNum(driver) {
-		return nil, errtypes.NotFound(driver)
+		return "", errtypes.NotFound(driver)
+	}
+
+	if isValidURL(driver) {
+		bin, err = downloadAndCompilePlugin(pluginType, driver)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		bin, err = checkDirAndCompile(pluginType, driver)
+		if err != nil {
+			return "", err
+		}
 	}
-	bin, err := checkDirAndCompile(pluginType, driver)
+	return bin, nil
+}
+
+// Load loads the plugin using the hashicorp go-plugin system
+func Load(pluginType, driver string) (*RevaPlugin, error) {
+	bin, err := fetchBinary(pluginType, driver)
 	if err != nil {
 		return nil, err
 	}
-
 	logger := hclog.New(&hclog.LoggerOptions{
 		Name:   "plugin",
 		Output: os.Stdout,
 		Level:  hclog.Trace,
 	})
 
 	client := plugin.NewClient(&plugin.ClientConfig{
-		HandshakeConfig: handshake,
+		HandshakeConfig: Handshake,
 		Plugins:         PluginMap,
 		Cmd:             exec.Command(bin),
 		AllowedProtocols: []plugin.Protocol{