feat: remove shared repo volume between repo-server and cmp-server

cmpserver/plugin/plugin.go

@@ -14,6 +14,8 @@ import (
 	"github.com/argoproj/pkg/rand"
 
 	"github.com/argoproj/argo-cd/v2/util/buffered_context"
+	"github.com/argoproj/argo-cd/v2/util/cmp"
+	"github.com/argoproj/argo-cd/v2/util/files"
 
 	"github.com/argoproj/gitops-engine/pkg/utils/kube"
 	"github.com/mattn/go-zglob"
@@ -128,7 +130,37 @@ func environ(envVars []*apiclient.EnvEntry) []string {
 }
 
 // GenerateManifest runs generate command from plugin config file and returns generated manifest files
-func (s *Service) GenerateManifest(ctx context.Context, q *apiclient.ManifestRequest) (*apiclient.ManifestResponse, error) {
+func (s *Service) GenerateManifest(stream apiclient.ConfigManagementPluginService_GenerateManifestServer) error {
+	workDir, err := files.CreateTempDir()
+	if err != nil {
+		return fmt.Errorf("error creating temp dir: %s", err)
+	}
+	defer func() {
+		if err := os.RemoveAll(workDir); err != nil {
+			// we panic here as the workDir may contain sensitive information
+			panic("error removing generate manifest workdir")
+		}
+	}()
+
+	metadata, err := cmp.ReceiveRepoStream(stream.Context(), stream, workDir)
+	if err != nil {
+		return fmt.Errorf("generate manifest error receiving stream: %s", err)
+	}
+
+	appPath := filepath.Clean(filepath.Join(workDir, metadata.AppRelPath))
+	response, err := s.generateManifest(stream.Context(), appPath, metadata.GetEnv())
+	if err != nil {
+		return fmt.Errorf("error generating manifests: %s", err)
+	}
+	err = stream.SendAndClose(response)
+	if err != nil {
+		return fmt.Errorf("error sending manifest response: %s", err)
+	}
+	return nil
+}
+
+// generateManifest runs generate command from plugin config file and returns generated manifest files
+func (s *Service) generateManifest(ctx context.Context, appDir string, envEntries []*apiclient.EnvEntry) (*apiclient.ManifestResponse, error) {
 	bufferedCtx, cancel := buffered_context.WithEarlierDeadline(ctx, cmpTimeoutBuffer)
 	defer cancel()
 
@@ -140,15 +172,15 @@ func (s *Service) GenerateManifest(ctx context.Context, q *apiclient.ManifestReq
 
 	config := s.initConstants.PluginConfig
 
-	env := append(os.Environ(), environ(q.Env)...)
+	env := append(os.Environ(), environ(envEntries)...)
 	if len(config.Spec.Init.Command) > 0 {
-		_, err := runCommand(bufferedCtx, config.Spec.Init, q.AppPath, env)
+		_, err := runCommand(bufferedCtx, config.Spec.Init, appDir, env)
 		if err != nil {
 			return &apiclient.ManifestResponse{}, err
 		}
 	}
 
-	out, err := runCommand(bufferedCtx, config.Spec.Generate, q.AppPath, env)
+	out, err := runCommand(bufferedCtx, config.Spec.Generate, appDir, env)
 	if err != nil {
 		return &apiclient.ManifestResponse{}, err
 	}
@@ -163,61 +195,85 @@ func (s *Service) GenerateManifest(ctx context.Context, q *apiclient.ManifestReq
 	}, err
 }
 
-// MatchRepository checks whether the application repository type is supported by config management plugin server
-func (s *Service) MatchRepository(ctx context.Context, q *apiclient.RepositoryRequest) (*apiclient.RepositoryResponse, error) {
-	bufferedCtx, cancel := buffered_context.WithEarlierDeadline(ctx, cmpTimeoutBuffer)
+// MatchRepository receives the application stream and checks whether
+// its repository type is supported by the config management plugin
+// server.
+//The checks are implemented in the following order:
+//   1. If spec.Discover.FileName is provided it finds for a name match in Applications files
+//   2. If spec.Discover.Find.Glob is provided if finds for a glob match in Applications files
+//   3. Otherwise it runs the spec.Discover.Find.Command
+func (s *Service) MatchRepository(stream apiclient.ConfigManagementPluginService_MatchRepositoryServer) error {
+	bufferedCtx, cancel := buffered_context.WithEarlierDeadline(stream.Context(), cmpTimeoutBuffer)
 	defer cancel()
 
-	var repoResponse apiclient.RepositoryResponse
+	workdir, err := files.CreateTempDir()
+	if err != nil {
+		return fmt.Errorf("error creating temp dir: %s", err)
+	}
+	defer func() {
+		if err := os.RemoveAll(workdir); err != nil {
+			log.Warnf("error removing workdir: %s", err)
+		}
+	}()
+
+	_, err = cmp.ReceiveRepoStream(bufferedCtx, stream, workdir)
+	if err != nil {
+		return fmt.Errorf("match repository error receiving stream: %s", err)
+	}
+
+	isSupported, err := s.matchRepository(bufferedCtx, workdir)
+	if err != nil {
+		return fmt.Errorf("match repository error: %s", err)
+	}
+	repoResponse := &apiclient.RepositoryResponse{IsSupported: isSupported}
+
+	err = stream.SendAndClose(repoResponse)
+	if err != nil {
+		return fmt.Errorf("error sending match repository response: %s", err)
+	}
+	return nil
+}
+
+func (s *Service) matchRepository(ctx context.Context, workdir string) (bool, error) {
 	config := s.initConstants.PluginConfig
 	if config.Spec.Discover.FileName != "" {
 		log.Debugf("config.Spec.Discover.FileName is provided")
-		pattern := strings.TrimSuffix(q.Path, "/") + "/" + strings.TrimPrefix(config.Spec.Discover.FileName, "/")
+		pattern := filepath.Join(workdir, config.Spec.Discover.FileName)
 		matches, err := filepath.Glob(pattern)
-		if err != nil || len(matches) == 0 {
-			log.Debugf("Could not find match for pattern %s. Error is %v.", pattern, err)
-			return &repoResponse, err
-		} else if len(matches) > 0 {
-			repoResponse.IsSupported = true
-			return &repoResponse, nil
+		if err != nil {
+			e := fmt.Errorf("error finding filename match for pattern %q: %s", pattern, err)
+			log.Debug(e)
+			return false, e
 		}
+		return len(matches) > 0, nil
 	}
 
 	if config.Spec.Discover.Find.Glob != "" {
 		log.Debugf("config.Spec.Discover.Find.Glob is provided")
-		pattern := strings.TrimSuffix(q.Path, "/") + "/" + strings.TrimPrefix(config.Spec.Discover.Find.Glob, "/")
+		pattern := filepath.Join(workdir, config.Spec.Discover.Find.Glob)
 		// filepath.Glob doesn't have '**' support hence selecting third-party lib
 		// https://github.com/golang/go/issues/11862
 		matches, err := zglob.Glob(pattern)
-		if err != nil || len(matches) == 0 {
-			log.Debugf("Could not find match for pattern %s. Error is %v.", pattern, err)
-			return &repoResponse, err
-		} else if len(matches) > 0 {
-			repoResponse.IsSupported = true
-			return &repoResponse, nil
+		if err != nil {
+			e := fmt.Errorf("error finding glob match for pattern %q: %s", pattern, err)
+			log.Debug(e)
+			return false, e
 		}
+
+		if len(matches) > 0 {
+			return true, nil
+		}
+		return false, nil
 	}
 
 	log.Debugf("Going to try runCommand.")
-	find, err := runCommand(bufferedCtx, config.Spec.Discover.Find.Command, q.Path, os.Environ())
+	find, err := runCommand(ctx, config.Spec.Discover.Find.Command, workdir, os.Environ())
 	if err != nil {
-		return &repoResponse, err
+		return false, fmt.Errorf("error running find command: %s", err)
 	}
 
-	var isSupported bool
 	if find != "" {
-		isSupported = true
+		return true, nil
 	}
-	return &apiclient.RepositoryResponse{
-		IsSupported: isSupported,
-	}, nil
-}
-
-// GetPluginConfig returns plugin config
-func (s *Service) GetPluginConfig(ctx context.Context, q *apiclient.ConfigRequest) (*apiclient.ConfigResponse, error) {
-	config := s.initConstants.PluginConfig
-	return &apiclient.ConfigResponse{
-		AllowConcurrency: config.Spec.AllowConcurrency,
-		LockRepo:         config.Spec.LockRepo,
-	}, nil
+	return false, nil
 }

cmpserver/plugin/plugin.proto

@@ -5,13 +5,27 @@ package plugin;
 
 import "k8s.io/api/core/v1/generated.proto";
 
-// ManifestRequest is a query for manifest generation.
-message ManifestRequest {
-    // Name of the application for which the request is triggered
+// AppStreamRequest is the request object used to send the application's
+// files over a stream.
+message AppStreamRequest {
+    oneof request {
+        ManifestRequestMetadata metadata = 1;
+        File file = 2;
+    }
+}
+
+// ManifestRequestMetadata defines the metada related to the file being sent
+// to the CMP server.
+message ManifestRequestMetadata {
+    // appName refers to the ArgoCD Application name
     string appName = 1;
-    string appPath = 2;
-    string repoPath = 3;
-    bool noCache = 4;
+    // appRelPath points to the application relative path inside the tarball
+    string appRelPath = 2;
+    // checksum is used to verify the integrity of the file
+    string checksum = 3;
+    // size relates to the file size in bytes
+    int64 size = 4;
+    // env is a list with the environment variables needed to generate manifests
     repeated EnvEntry env = 5;
 }
 
@@ -28,34 +42,22 @@ message ManifestResponse {
     string sourceType = 2;
 }
 
-message RepositoryRequest {
-    string path = 1;
-    repeated EnvEntry env = 2;
-}
-
 message RepositoryResponse {
     bool isSupported = 1;
 }
 
-message ConfigRequest {
-}
-
-message ConfigResponse {
-    bool allowConcurrency = 1;
-    bool lockRepo = 2;
+message File {
+    bytes chunk = 1;
 }
 
 // ConfigManagementPlugin Service
 service ConfigManagementPluginService {
-    // GenerateManifest generates manifest for application in specified repo name and revision
-    rpc GenerateManifest(ManifestRequest) returns (ManifestResponse) {
-    }
-
-    // MatchRepository returns whether or not the given path is supported by the plugin
-    rpc MatchRepository(RepositoryRequest) returns (RepositoryResponse) {
+    // GenerateManifests receive a stream containing a tgz archive with all required files necessary
+    // to generate manifests
+    rpc GenerateManifest(stream AppStreamRequest) returns (ManifestResponse) {
     }
 
-    // Get configuration of the plugin
-    rpc GetPluginConfig(ConfigRequest) returns (ConfigResponse) {
+    // MatchRepository returns whether or not the given application is supported by the plugin
+    rpc MatchRepository(stream AppStreamRequest) returns (RepositoryResponse) {
     }
 }