Skip to content

Commit

Permalink
feat: remove shared repo volume between repo-server and cmp-server (#…
Browse files Browse the repository at this point in the history 
…8600)

feat: remove shared repo volume between repo-server and cmp-server (#8600)

Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com>
  • Loading branch information
@leoluz
leoluz authored Mar 15, 2022
1 parent 48cf001 commit 41db812
Show file tree
Hide file tree
Showing 28 changed files with 2,158 additions and 818 deletions.
1,276 changes: 617 additions & 659 deletions cmpserver/apiclient/plugin.pb.go
View file

Large diffs are not rendered by default.

153 changes: 113 additions & 40 deletions cmpserver/plugin/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@ import (

"github.com/argoproj/pkg/rand"

"github.com/argoproj/argo-cd/v2/common"
"github.com/argoproj/argo-cd/v2/util/buffered_context"
"github.com/argoproj/argo-cd/v2/util/cmp"
"github.com/argoproj/argo-cd/v2/util/io/files"

"github.com/argoproj/gitops-engine/pkg/utils/kube"
"github.com/mattn/go-zglob"
Expand Down Expand Up @@ -42,6 +45,19 @@ func NewService(initConstants CMPServerInitConstants) *Service {
}
}

func (s *Service) Init() error {
workDir := common.GetCMPWorkDir()
err := os.RemoveAll(workDir)
if err != nil {
return fmt.Errorf("error removing workdir %q: %s", workDir, err)
}
err = os.MkdirAll(workDir, 0700)
if err != nil {
return fmt.Errorf("error creating workdir %q: %s", workDir, err)
}
return nil
}

func runCommand(ctx context.Context, command Command, path string, env []string) (string, error) {
if len(command.Command) == 0 {
return "", fmt.Errorf("Command is empty")
Expand Down Expand Up @@ -128,27 +144,59 @@ func environ(envVars []*apiclient.EnvEntry) []string {
}

// GenerateManifest runs generate command from plugin config file and returns generated manifest files
func (s *Service) GenerateManifest(ctx context.Context, q *apiclient.ManifestRequest) (*apiclient.ManifestResponse, error) {
bufferedCtx, cancel := buffered_context.WithEarlierDeadline(ctx, cmpTimeoutBuffer)
func (s *Service) GenerateManifest(stream apiclient.ConfigManagementPluginService_GenerateManifestServer) error {
ctx, cancel := buffered_context.WithEarlierDeadline(stream.Context(), cmpTimeoutBuffer)
defer cancel()
workDir, err := files.CreateTempDir(common.GetCMPWorkDir())
if err != nil {
return fmt.Errorf("error creating temp dir: %s", err)
}
defer func() {
if err := os.RemoveAll(workDir); err != nil {
// we panic here as the workDir may contain sensitive information
panic(fmt.Sprintf("error removing generate manifest workdir: %s", err))
}
}()

if deadline, ok := bufferedCtx.Deadline(); ok {
metadata, err := cmp.ReceiveRepoStream(ctx, stream, workDir)
if err != nil {
return fmt.Errorf("generate manifest error receiving stream: %s", err)
}

appPath := filepath.Clean(filepath.Join(workDir, metadata.AppRelPath))
if !strings.HasPrefix(appPath, workDir) {
return fmt.Errorf("illegal appPath: out of workDir bound")
}
response, err := s.generateManifest(ctx, appPath, metadata.GetEnv())
if err != nil {
return fmt.Errorf("error generating manifests: %s", err)
}
err = stream.SendAndClose(response)
if err != nil {
return fmt.Errorf("error sending manifest response: %s", err)
}
return nil
}

// generateManifest runs generate command from plugin config file and returns generated manifest files
func (s *Service) generateManifest(ctx context.Context, appDir string, envEntries []*apiclient.EnvEntry) (*apiclient.ManifestResponse, error) {
if deadline, ok := ctx.Deadline(); ok {
log.Infof("Generating manifests with deadline %v from now", time.Until(deadline))
} else {
log.Info("Generating manifests with no request-level timeout")
}

config := s.initConstants.PluginConfig

env := append(os.Environ(), environ(q.Env)...)
env := append(os.Environ(), environ(envEntries)...)
if len(config.Spec.Init.Command) > 0 {
_, err := runCommand(bufferedCtx, config.Spec.Init, q.AppPath, env)
_, err := runCommand(ctx, config.Spec.Init, appDir, env)
if err != nil {
return &apiclient.ManifestResponse{}, err
}
}

out, err := runCommand(bufferedCtx, config.Spec.Generate, q.AppPath, env)
out, err := runCommand(ctx, config.Spec.Generate, appDir, env)
if err != nil {
return &apiclient.ManifestResponse{}, err
}
Expand All @@ -163,61 +211,86 @@ func (s *Service) GenerateManifest(ctx context.Context, q *apiclient.ManifestReq
}, err
}

// MatchRepository checks whether the application repository type is supported by config management plugin server
func (s *Service) MatchRepository(ctx context.Context, q *apiclient.RepositoryRequest) (*apiclient.RepositoryResponse, error) {
bufferedCtx, cancel := buffered_context.WithEarlierDeadline(ctx, cmpTimeoutBuffer)
// MatchRepository receives the application stream and checks whether
// its repository type is supported by the config management plugin
// server.
//The checks are implemented in the following order:
// 1. If spec.Discover.FileName is provided it finds for a name match in Applications files
// 2. If spec.Discover.Find.Glob is provided if finds for a glob match in Applications files
// 3. Otherwise it runs the spec.Discover.Find.Command
func (s *Service) MatchRepository(stream apiclient.ConfigManagementPluginService_MatchRepositoryServer) error {
bufferedCtx, cancel := buffered_context.WithEarlierDeadline(stream.Context(), cmpTimeoutBuffer)
defer cancel()

var repoResponse apiclient.RepositoryResponse
workDir, err := files.CreateTempDir(common.GetCMPWorkDir())
if err != nil {
return fmt.Errorf("error creating match repository workdir: %s", err)
}
defer func() {
if err := os.RemoveAll(workDir); err != nil {
// we panic here as the workDir may contain sensitive information
panic(fmt.Sprintf("error removing match repository workdir: %s", err))
}
}()

_, err = cmp.ReceiveRepoStream(bufferedCtx, stream, workDir)
if err != nil {
return fmt.Errorf("match repository error receiving stream: %s", err)
}

isSupported, err := s.matchRepository(bufferedCtx, workDir)
if err != nil {
return fmt.Errorf("match repository error: %s", err)
}
repoResponse := &apiclient.RepositoryResponse{IsSupported: isSupported}

err = stream.SendAndClose(repoResponse)
if err != nil {
return fmt.Errorf("error sending match repository response: %s", err)
}
return nil
}

func (s *Service) matchRepository(ctx context.Context, workdir string) (bool, error) {
config := s.initConstants.PluginConfig
if config.Spec.Discover.FileName != "" {
log.Debugf("config.Spec.Discover.FileName is provided")
pattern := strings.TrimSuffix(q.Path, "/") + "/" + strings.TrimPrefix(config.Spec.Discover.FileName, "/")
pattern := filepath.Join(workdir, config.Spec.Discover.FileName)
matches, err := filepath.Glob(pattern)
if err != nil || len(matches) == 0 {
log.Debugf("Could not find match for pattern %s. Error is %v.", pattern, err)
return &repoResponse, err
} else if len(matches) > 0 {
repoResponse.IsSupported = true
return &repoResponse, nil
if err != nil {
e := fmt.Errorf("error finding filename match for pattern %q: %s", pattern, err)
log.Debug(e)
return false, e
}
return len(matches) > 0, nil
}

if config.Spec.Discover.Find.Glob != "" {
log.Debugf("config.Spec.Discover.Find.Glob is provided")
pattern := strings.TrimSuffix(q.Path, "/") + "/" + strings.TrimPrefix(config.Spec.Discover.Find.Glob, "/")
pattern := filepath.Join(workdir, config.Spec.Discover.Find.Glob)
// filepath.Glob doesn't have '**' support hence selecting third-party lib
// https://github.com/golang/go/issues/11862
matches, err := zglob.Glob(pattern)
if err != nil || len(matches) == 0 {
log.Debugf("Could not find match for pattern %s. Error is %v.", pattern, err)
return &repoResponse, err
} else if len(matches) > 0 {
repoResponse.IsSupported = true
return &repoResponse, nil
if err != nil {
e := fmt.Errorf("error finding glob match for pattern %q: %s", pattern, err)
log.Debug(e)
return false, e
}

if len(matches) > 0 {
return true, nil
}
return false, nil
}

log.Debugf("Going to try runCommand.")
find, err := runCommand(bufferedCtx, config.Spec.Discover.Find.Command, q.Path, os.Environ())
find, err := runCommand(ctx, config.Spec.Discover.Find.Command, workdir, os.Environ())
if err != nil {
return &repoResponse, err
return false, fmt.Errorf("error running find command: %s", err)
}

var isSupported bool
if find != "" {
isSupported = true
return true, nil
}
return &apiclient.RepositoryResponse{
IsSupported: isSupported,
}, nil
}

// GetPluginConfig returns plugin config
func (s *Service) GetPluginConfig(ctx context.Context, q *apiclient.ConfigRequest) (*apiclient.ConfigResponse, error) {
config := s.initConstants.PluginConfig
return &apiclient.ConfigResponse{
AllowConcurrency: config.Spec.AllowConcurrency,
LockRepo: config.Spec.LockRepo,
}, nil
return false, nil
}
52 changes: 27 additions & 25 deletions cmpserver/plugin/plugin.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,27 @@ package plugin;

import "k8s.io/api/core/v1/generated.proto";

// ManifestRequest is a query for manifest generation.
message ManifestRequest {
// Name of the application for which the request is triggered
// AppStreamRequest is the request object used to send the application's
// files over a stream.
message AppStreamRequest {
oneof request {
ManifestRequestMetadata metadata = 1;
File file = 2;
}
}

// ManifestRequestMetadata defines the metada related to the file being sent
// to the CMP server.
message ManifestRequestMetadata {
// appName refers to the ArgoCD Application name
string appName = 1;
string appPath = 2;
string repoPath = 3;
bool noCache = 4;
// appRelPath points to the application relative path inside the tarball
string appRelPath = 2;
// checksum is used to verify the integrity of the file
string checksum = 3;
// size relates to the file size in bytes
int64 size = 4;
// env is a list with the environment variables needed to generate manifests
repeated EnvEntry env = 5;
}

Expand All @@ -28,34 +42,22 @@ message ManifestResponse {
string sourceType = 2;
}

message RepositoryRequest {
string path = 1;
repeated EnvEntry env = 2;
}

message RepositoryResponse {
bool isSupported = 1;
}

message ConfigRequest {
}

message ConfigResponse {
bool allowConcurrency = 1;
bool lockRepo = 2;
message File {
bytes chunk = 1;
}

// ConfigManagementPlugin Service
service ConfigManagementPluginService {
// GenerateManifest generates manifest for application in specified repo name and revision
rpc GenerateManifest(ManifestRequest) returns (ManifestResponse) {
}

// MatchRepository returns whether or not the given path is supported by the plugin
rpc MatchRepository(RepositoryRequest) returns (RepositoryResponse) {
// GenerateManifests receive a stream containing a tgz archive with all required files necessary
// to generate manifests
rpc GenerateManifest(stream AppStreamRequest) returns (ManifestResponse) {
}

// Get configuration of the plugin
rpc GetPluginConfig(ConfigRequest) returns (ConfigResponse) {
// MatchRepository returns whether or not the given application is supported by the plugin
rpc MatchRepository(stream AppStreamRequest) returns (RepositoryResponse) {
}
}
Loading

0 comments on commit 41db812

Please sign in to comment.