From 88804106cd60922d7e069ab724b1ac45f1ec6cb3 Mon Sep 17 00:00:00 2001 From: Leonardo Luz Almeida Date: Thu, 10 Mar 2022 16:25:54 -0500 Subject: [PATCH] Send repo folder to CMP instead of just app folder Signed-off-by: Leonardo Luz Almeida --- cmpserver/apiclient/plugin.pb.go | 135 ++++++++++++++++++---------- cmpserver/plugin/plugin.go | 13 +-- reposerver/repository/repository.go | 17 ++-- test/cmp/plugin.yaml | 3 +- util/app/discovery/discovery.go | 22 ++--- util/cmp/stream.go | 40 +++++---- util/cmp/stream_test.go | 4 +- 7 files changed, 146 insertions(+), 88 deletions(-) diff --git a/cmpserver/apiclient/plugin.pb.go b/cmpserver/apiclient/plugin.pb.go index 3cb0242e33d62..bcffb01cf55bc 100644 --- a/cmpserver/apiclient/plugin.pb.go +++ b/cmpserver/apiclient/plugin.pb.go @@ -117,11 +117,19 @@ func (*AppStreamRequest) XXX_OneofWrappers() []interface{} { } } +// ManifestRequestMetadata defines the metada related to the file being sent +// to the CMP server. type ManifestRequestMetadata struct { - AppName string `protobuf:"bytes,1,opt,name=appName,proto3" json:"appName,omitempty"` - AppPath string `protobuf:"bytes,2,opt,name=appPath,proto3" json:"appPath,omitempty"` - Checksum string `protobuf:"bytes,3,opt,name=checksum,proto3" json:"checksum,omitempty"` - Env []*EnvEntry `protobuf:"bytes,4,rep,name=env,proto3" json:"env,omitempty"` + // appName refers to the ArgoCD Application name + AppName string `protobuf:"bytes,1,opt,name=appName,proto3" json:"appName,omitempty"` + // appRelPath points to the application relative path inside the tarball + AppRelPath string `protobuf:"bytes,2,opt,name=appRelPath,proto3" json:"appRelPath,omitempty"` + // checksum is used to verify the integrity of the file + Checksum string `protobuf:"bytes,3,opt,name=checksum,proto3" json:"checksum,omitempty"` + // size relates to the file size in bytes + Size_ int64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` + // env is a list with the environment variables needed to generate manifests + Env []*EnvEntry `protobuf:"bytes,5,rep,name=env,proto3" json:"env,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -167,9 +175,9 @@ func (m *ManifestRequestMetadata) GetAppName() string { return "" } -func (m *ManifestRequestMetadata) GetAppPath() string { +func (m *ManifestRequestMetadata) GetAppRelPath() string { if m != nil { - return m.AppPath + return m.AppRelPath } return "" } @@ -181,6 +189,13 @@ func (m *ManifestRequestMetadata) GetChecksum() string { return "" } +func (m *ManifestRequestMetadata) GetSize_() int64 { + if m != nil { + return m.Size_ + } + return 0 +} + func (m *ManifestRequestMetadata) GetEnv() []*EnvEntry { if m != nil { return m.Env @@ -407,38 +422,39 @@ func init() { func init() { proto.RegisterFile("cmpserver/plugin/plugin.proto", fileDescriptor_b21875a7079a06ed) } var fileDescriptor_b21875a7079a06ed = []byte{ - // 488 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x53, 0xc1, 0x8e, 0x12, 0x4d, - 0x10, 0xa6, 0x7f, 0xf8, 0x77, 0xa1, 0xd8, 0x44, 0xd2, 0x31, 0x71, 0x42, 0x76, 0x91, 0xf4, 0x89, - 0x8b, 0x4c, 0x44, 0x63, 0xbc, 0x98, 0xe8, 0x9a, 0xd5, 0x8d, 0x06, 0x43, 0x06, 0x4f, 0xde, 0x7a, - 0x9b, 0x62, 0x68, 0x61, 0xba, 0xdb, 0xee, 0x9e, 0x49, 0xb8, 0xf9, 0x02, 0xbe, 0x8f, 0x8f, 0xe0, - 0xd1, 0x47, 0x30, 0x3c, 0x89, 0xa1, 0x67, 0x06, 0x36, 0x6e, 0xf4, 0x34, 0xfd, 0xd5, 0x57, 0xf5, - 0xd5, 0x57, 0x95, 0x1a, 0xb8, 0x10, 0x99, 0x71, 0x68, 0x0b, 0xb4, 0xb1, 0xd9, 0xe4, 0xa9, 0x54, - 0xd5, 0x67, 0x6c, 0xac, 0xf6, 0x9a, 0x9e, 0x94, 0xa8, 0xcf, 0xd6, 0xcf, 0xdd, 0x58, 0xea, 0x98, - 0x1b, 0x19, 0x0b, 0x6d, 0x31, 0x2e, 0x1e, 0xc7, 0x29, 0x2a, 0xb4, 0xdc, 0xe3, 0xa2, 0xcc, 0x65, - 0x5f, 0x09, 0xf4, 0x5e, 0x19, 0x33, 0xf7, 0x16, 0x79, 0x96, 0xe0, 0x97, 0x1c, 0x9d, 0xa7, 0x2f, - 0xa0, 0x9d, 0xa1, 0xe7, 0x0b, 0xee, 0x79, 0x44, 0x86, 0x64, 0xd4, 0x9d, 0x3c, 0x1c, 0x57, 0x1d, - 0xa6, 0x5c, 0xc9, 0x25, 0x3a, 0x5f, 0xa5, 0x4e, 0xab, 0xb4, 0xeb, 0x46, 0x72, 0x28, 0xa1, 0x0c, - 0x5a, 0x4b, 0xb9, 0xc1, 0xe8, 0xbf, 0x50, 0x7a, 0x56, 0x97, 0xbe, 0x91, 0x1b, 0xbc, 0x6e, 0x24, - 0x81, 0xbb, 0xec, 0xc0, 0xa9, 0x2d, 0x25, 0xd8, 0x37, 0x02, 0x0f, 0xfe, 0x22, 0x4b, 0x23, 0x38, - 0xe5, 0xc6, 0x7c, 0xe0, 0x19, 0x06, 0x23, 0x9d, 0xa4, 0x86, 0x15, 0x33, 0xe3, 0x7e, 0x15, 0xfa, - 0x94, 0xcc, 0x1e, 0xd2, 0x3e, 0xb4, 0xc5, 0x0a, 0xc5, 0xda, 0xe5, 0x59, 0xd4, 0x0c, 0xd4, 0x01, - 0x53, 0x06, 0x4d, 0x54, 0x45, 0xd4, 0x1a, 0x36, 0x47, 0xdd, 0x49, 0xaf, 0x76, 0x76, 0xa5, 0x8a, - 0x2b, 0xe5, 0xed, 0x36, 0xd9, 0x93, 0xec, 0x29, 0xb4, 0xeb, 0x00, 0xa5, 0xd0, 0x52, 0xc7, 0xe6, - 0xe1, 0x4d, 0xef, 0xc3, 0xff, 0x05, 0xdf, 0xe4, 0x58, 0xf5, 0x2d, 0x01, 0x9b, 0x41, 0xef, 0x38, - 0x84, 0x33, 0x5a, 0x39, 0xa4, 0xe7, 0xd0, 0xc9, 0xaa, 0x98, 0x8b, 0xc8, 0xb0, 0x39, 0xea, 0x24, - 0xc7, 0x00, 0x1d, 0x00, 0x38, 0x9d, 0x5b, 0x81, 0x1f, 0xb7, 0xa6, 0x16, 0xbb, 0x15, 0x61, 0xcf, - 0x80, 0x26, 0x68, 0xb4, 0x93, 0x5e, 0xdb, 0xed, 0x41, 0x73, 0x08, 0x5d, 0xe9, 0xe6, 0xb9, 0x31, - 0xda, 0x7a, 0x5c, 0x04, 0x63, 0xed, 0xe4, 0x76, 0x88, 0x9d, 0x43, 0x6b, 0xbf, 0xea, 0xbd, 0x4f, - 0xb1, 0xca, 0xd5, 0x3a, 0xe4, 0x9c, 0x25, 0x25, 0x98, 0x7c, 0x27, 0x70, 0xf1, 0x5a, 0xab, 0xa5, - 0x4c, 0xa7, 0x5c, 0xf1, 0x14, 0x33, 0x54, 0x7e, 0x16, 0xd6, 0x30, 0x47, 0x5b, 0x48, 0x81, 0xf4, - 0x1d, 0xf4, 0xde, 0x56, 0x57, 0x52, 0x4f, 0x44, 0xa3, 0x7a, 0x55, 0x7f, 0xde, 0x4a, 0x3f, 0xba, - 0x7b, 0x19, 0xa5, 0x53, 0xd6, 0x18, 0x11, 0xfa, 0x1e, 0xee, 0x4d, 0xb9, 0x17, 0xab, 0xe3, 0x20, - 0xff, 0x90, 0xea, 0xd7, 0xcc, 0xdd, 0xb1, 0xf7, 0x62, 0x97, 0x2f, 0x7f, 0xec, 0x06, 0xe4, 0xe7, - 0x6e, 0x40, 0x7e, 0xed, 0x06, 0xe4, 0xd3, 0x24, 0x95, 0x7e, 0x95, 0xdf, 0x8c, 0x85, 0xce, 0x62, - 0x6e, 0x53, 0x6d, 0xac, 0xfe, 0x1c, 0x1e, 0x8f, 0xc4, 0x22, 0x2e, 0x26, 0xf1, 0xf1, 0x1f, 0xe1, - 0x46, 0x8a, 0x8d, 0x44, 0xe5, 0x6f, 0x4e, 0xc2, 0xd1, 0x3f, 0xf9, 0x1d, 0x00, 0x00, 0xff, 0xff, - 0xd0, 0x9f, 0xb1, 0x3c, 0x41, 0x03, 0x00, 0x00, + // 506 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x53, 0xc1, 0x8e, 0xd3, 0x3c, + 0x10, 0xde, 0xfc, 0xed, 0xee, 0xb6, 0xb3, 0x2b, 0xfd, 0x95, 0x85, 0x44, 0x54, 0xed, 0x96, 0x2a, + 0xa7, 0x5e, 0x68, 0x44, 0x41, 0x88, 0x0b, 0x12, 0x2c, 0x5a, 0x58, 0x81, 0x8a, 0x2a, 0x97, 0x13, + 0x37, 0xaf, 0x3b, 0x4d, 0x4d, 0x13, 0xdb, 0xd8, 0x4e, 0xa4, 0x72, 0xe2, 0x6d, 0x78, 0x05, 0x1e, + 0x81, 0x23, 0x8f, 0x80, 0xfa, 0x24, 0x28, 0x4e, 0xd2, 0x56, 0xac, 0xe0, 0x94, 0x99, 0x6f, 0x66, + 0x3e, 0x7f, 0x9f, 0x33, 0x86, 0x4b, 0x9e, 0x69, 0x8b, 0xa6, 0x40, 0x13, 0xeb, 0x34, 0x4f, 0x84, + 0xac, 0x3f, 0x63, 0x6d, 0x94, 0x53, 0xe4, 0xa4, 0xca, 0xfa, 0xd1, 0xfa, 0x99, 0x1d, 0x0b, 0x15, + 0x33, 0x2d, 0x62, 0xae, 0x0c, 0xc6, 0xc5, 0xa3, 0x38, 0x41, 0x89, 0x86, 0x39, 0x5c, 0x54, 0xbd, + 0xd1, 0xd7, 0x00, 0x7a, 0x2f, 0xb5, 0x9e, 0x3b, 0x83, 0x2c, 0xa3, 0xf8, 0x39, 0x47, 0xeb, 0xc8, + 0x73, 0xe8, 0x64, 0xe8, 0xd8, 0x82, 0x39, 0x16, 0x06, 0xc3, 0x60, 0x74, 0x36, 0x79, 0x30, 0xae, + 0x4f, 0x98, 0x32, 0x29, 0x96, 0x68, 0x5d, 0xdd, 0x3a, 0xad, 0xdb, 0x6e, 0x8e, 0xe8, 0x6e, 0x84, + 0x44, 0xd0, 0x5e, 0x8a, 0x14, 0xc3, 0xff, 0xfc, 0xe8, 0x79, 0x33, 0xfa, 0x5a, 0xa4, 0x78, 0x73, + 0x44, 0x7d, 0xed, 0xaa, 0x0b, 0xa7, 0xa6, 0xa2, 0x88, 0xbe, 0x05, 0x70, 0xff, 0x2f, 0xb4, 0x24, + 0x84, 0x53, 0xa6, 0xf5, 0x7b, 0x96, 0xa1, 0x17, 0xd2, 0xa5, 0x4d, 0x4a, 0x06, 0x00, 0x4c, 0x6b, + 0x8a, 0xe9, 0x8c, 0xb9, 0x95, 0x3f, 0xaa, 0x4b, 0x0f, 0x10, 0xd2, 0x87, 0x0e, 0x5f, 0x21, 0x5f, + 0xdb, 0x3c, 0x0b, 0x5b, 0xbe, 0xba, 0xcb, 0x09, 0x81, 0xb6, 0x15, 0x5f, 0x30, 0x6c, 0x0f, 0x83, + 0x51, 0x8b, 0xfa, 0x98, 0x44, 0xd0, 0x42, 0x59, 0x84, 0xc7, 0xc3, 0xd6, 0xe8, 0x6c, 0xd2, 0x6b, + 0x34, 0x5f, 0xcb, 0xe2, 0x5a, 0x3a, 0xb3, 0xa1, 0x65, 0x31, 0x7a, 0x02, 0x9d, 0x06, 0x28, 0x39, + 0xe4, 0x5e, 0x96, 0x8f, 0xc9, 0x3d, 0x38, 0x2e, 0x58, 0x9a, 0x63, 0x2d, 0xa7, 0x4a, 0xa2, 0x19, + 0xf4, 0xf6, 0xf6, 0xac, 0x56, 0xd2, 0x22, 0xb9, 0x80, 0x6e, 0x56, 0x63, 0x36, 0x0c, 0x86, 0xad, + 0x51, 0x97, 0xee, 0x81, 0xd2, 0x9b, 0x55, 0xb9, 0xe1, 0xf8, 0x61, 0xa3, 0x1b, 0xb2, 0x03, 0x24, + 0x7a, 0x0a, 0x84, 0xa2, 0x56, 0x56, 0x38, 0x65, 0x36, 0x3b, 0xce, 0x21, 0x9c, 0x09, 0x3b, 0xcf, + 0xb5, 0x56, 0xc6, 0xe1, 0xc2, 0x0b, 0xeb, 0xd0, 0x43, 0x28, 0xba, 0x80, 0x76, 0xf9, 0x13, 0x4a, + 0x9d, 0x7c, 0x95, 0xcb, 0xb5, 0xef, 0x39, 0xa7, 0x55, 0x32, 0xf9, 0x1e, 0xc0, 0xe5, 0x2b, 0x25, + 0x97, 0x22, 0x99, 0x32, 0xc9, 0x12, 0xcc, 0x50, 0xba, 0x99, 0xbf, 0x86, 0x39, 0x9a, 0x42, 0x70, + 0x24, 0x6f, 0xa1, 0xf7, 0xa6, 0xde, 0x9f, 0xc6, 0x11, 0x09, 0x9b, 0xab, 0xfa, 0x73, 0x8b, 0xfa, + 0xe1, 0xdd, 0x9d, 0xa9, 0x94, 0x46, 0x47, 0xa3, 0x80, 0xbc, 0x83, 0xff, 0xa7, 0xcc, 0xf1, 0xd5, + 0xde, 0xc8, 0x3f, 0xa8, 0xfa, 0x4d, 0xe5, 0xae, 0xed, 0x92, 0xec, 0xea, 0xc5, 0x8f, 0xed, 0x20, + 0xf8, 0xb9, 0x1d, 0x04, 0xbf, 0xb6, 0x83, 0xe0, 0xe3, 0x24, 0x11, 0x6e, 0x95, 0xdf, 0x8e, 0xb9, + 0xca, 0x62, 0x66, 0x12, 0xa5, 0x8d, 0xfa, 0xe4, 0x83, 0x87, 0x7c, 0x11, 0x17, 0x93, 0x78, 0xff, + 0x7a, 0x98, 0x16, 0x3c, 0x15, 0x28, 0xdd, 0xed, 0x89, 0x7f, 0x0e, 0x8f, 0x7f, 0x07, 0x00, 0x00, + 0xff, 0xff, 0x75, 0x01, 0x97, 0x9d, 0x5b, 0x03, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -744,9 +760,14 @@ func (m *ManifestRequestMetadata) MarshalToSizedBuffer(dAtA []byte) (int, error) i = encodeVarintPlugin(dAtA, i, uint64(size)) } i-- - dAtA[i] = 0x22 + dAtA[i] = 0x2a } } + if m.Size_ != 0 { + i = encodeVarintPlugin(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x20 + } if len(m.Checksum) > 0 { i -= len(m.Checksum) copy(dAtA[i:], m.Checksum) @@ -754,10 +775,10 @@ func (m *ManifestRequestMetadata) MarshalToSizedBuffer(dAtA []byte) (int, error) i-- dAtA[i] = 0x1a } - if len(m.AppPath) > 0 { - i -= len(m.AppPath) - copy(dAtA[i:], m.AppPath) - i = encodeVarintPlugin(dAtA, i, uint64(len(m.AppPath))) + if len(m.AppRelPath) > 0 { + i -= len(m.AppRelPath) + copy(dAtA[i:], m.AppRelPath) + i = encodeVarintPlugin(dAtA, i, uint64(len(m.AppRelPath))) i-- dAtA[i] = 0x12 } @@ -986,7 +1007,7 @@ func (m *ManifestRequestMetadata) Size() (n int) { if l > 0 { n += 1 + l + sovPlugin(uint64(l)) } - l = len(m.AppPath) + l = len(m.AppRelPath) if l > 0 { n += 1 + l + sovPlugin(uint64(l)) } @@ -994,6 +1015,9 @@ func (m *ManifestRequestMetadata) Size() (n int) { if l > 0 { n += 1 + l + sovPlugin(uint64(l)) } + if m.Size_ != 0 { + n += 1 + sovPlugin(uint64(m.Size_)) + } if len(m.Env) > 0 { for _, e := range m.Env { l = e.Size() @@ -1269,7 +1293,7 @@ func (m *ManifestRequestMetadata) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field AppPath", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field AppRelPath", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -1297,7 +1321,7 @@ func (m *ManifestRequestMetadata) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.AppPath = string(dAtA[iNdEx:postIndex]) + m.AppRelPath = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { @@ -1332,6 +1356,25 @@ func (m *ManifestRequestMetadata) Unmarshal(dAtA []byte) error { m.Checksum = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlugin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Env", wireType) } diff --git a/cmpserver/plugin/plugin.go b/cmpserver/plugin/plugin.go index c109116baf045..bdfa44d14a103 100644 --- a/cmpserver/plugin/plugin.go +++ b/cmpserver/plugin/plugin.go @@ -141,12 +141,13 @@ func (s *Service) GenerateManifest(stream apiclient.ConfigManagementPluginServic } }() - env, err := cmp.ReceiveApplicationStream(stream.Context(), stream, workDir) + metadata, err := cmp.ReceiveRepoStream(stream.Context(), stream, workDir) if err != nil { return fmt.Errorf("generate manifest error receiving stream: %s", err) } - response, err := s.generateManifest(stream.Context(), workDir, env) + appPath := filepath.Clean(filepath.Join(workDir, metadata.AppRelPath)) + response, err := s.generateManifest(stream.Context(), appPath, metadata.GetEnv()) if err != nil { return fmt.Errorf("error generating manifests: %s", err) } @@ -158,7 +159,7 @@ func (s *Service) GenerateManifest(stream apiclient.ConfigManagementPluginServic } // generateManifest runs generate command from plugin config file and returns generated manifest files -func (s *Service) generateManifest(ctx context.Context, workDir string, envEntries []*apiclient.EnvEntry) (*apiclient.ManifestResponse, error) { +func (s *Service) generateManifest(ctx context.Context, appDir string, envEntries []*apiclient.EnvEntry) (*apiclient.ManifestResponse, error) { bufferedCtx, cancel := buffered_context.WithEarlierDeadline(ctx, cmpTimeoutBuffer) defer cancel() @@ -172,13 +173,13 @@ func (s *Service) generateManifest(ctx context.Context, workDir string, envEntri env := append(os.Environ(), environ(envEntries)...) if len(config.Spec.Init.Command) > 0 { - _, err := runCommand(bufferedCtx, config.Spec.Init, workDir, env) + _, err := runCommand(bufferedCtx, config.Spec.Init, appDir, env) if err != nil { return &apiclient.ManifestResponse{}, err } } - out, err := runCommand(bufferedCtx, config.Spec.Generate, workDir, env) + out, err := runCommand(bufferedCtx, config.Spec.Generate, appDir, env) if err != nil { return &apiclient.ManifestResponse{}, err } @@ -214,7 +215,7 @@ func (s *Service) MatchRepository(stream apiclient.ConfigManagementPluginService } }() - _, err = cmp.ReceiveApplicationStream(bufferedCtx, stream, workdir) + _, err = cmp.ReceiveRepoStream(bufferedCtx, stream, workdir) if err != nil { return fmt.Errorf("match repository error receiving stream: %s", err) } diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go index e03be8d17bc04..9c2e426891611 100644 --- a/reposerver/repository/repository.go +++ b/reposerver/repository/repository.go @@ -384,7 +384,12 @@ func (s *Service) GenerateManifest(ctx context.Context, q *apiclient.ManifestReq err = s.runRepoOperation(ctx, q.Revision, q.Repo, q.ApplicationSource, q.VerifySignature, cacheFn, operation, settings) if tarConcluded && res == nil { - res = <-promise.responseCh + select { + case resp := <-promise.responseCh: + res = resp + case err := <-promise.errCh: + return nil, err + } } return res, err @@ -473,7 +478,6 @@ func (s *Service) runManifestGenAsync(ctx context.Context, repoRoot, commitSHA, } ch.errCh <- err - return } // Otherwise, no error occurred, so ensure the manifest generation error data in the cache entry is reset before we cache the value manifestGenCacheEntry := cache.CachedManifestResponse{ @@ -490,7 +494,6 @@ func (s *Service) runManifestGenAsync(ctx context.Context, repoRoot, commitSHA, log.Warnf("manifest cache set error %s/%s: %v", q.ApplicationSource.String(), cacheKey, err) } ch.responseCh <- manifestGenCacheEntry.ManifestResponse - return } // getManifestCacheEntry returns false if the 'generate manifests' operation should be run by runRepoOperation, e.g.: @@ -1322,7 +1325,7 @@ func getPluginEnvs(envVars *v1alpha1.Env, q *apiclient.ManifestRequest, creds gi func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath string, envVars *v1alpha1.Env, q *apiclient.ManifestRequest, creds git.Creds, tarDoneCh chan<- bool) ([]*unstructured.Unstructured, error) { // detect config management plugin server (sidecar) - conn, cmpClient, err := discovery.DetectConfigManagementPlugin(ctx, appPath) + conn, cmpClient, err := discovery.DetectConfigManagementPlugin(ctx, repoPath) if err != nil { return nil, err } @@ -1333,7 +1336,7 @@ func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath st if err != nil { return nil, err } - cmpManifests, err := generateManifestsCMP(ctx, appPath, env, cmpClient, tarDoneCh) + cmpManifests, err := generateManifestsCMP(ctx, appPath, repoPath, env, cmpClient, tarDoneCh) if err != nil { return nil, fmt.Errorf("error generating manifests in cmp: %s", err) } @@ -1351,7 +1354,7 @@ func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath st // generateManifestsCMP will send the appPath files to the cmp-server over a gRPC stream. // The cmp-server will generate the manifests. Returns a response object with the generated // manifests. -func generateManifestsCMP(ctx context.Context, appPath string, env []string, cmpClient pluginclient.ConfigManagementPluginServiceClient, tarDoneCh chan<- bool) (*pluginclient.ManifestResponse, error) { +func generateManifestsCMP(ctx context.Context, appPath, repoPath string, env []string, cmpClient pluginclient.ConfigManagementPluginServiceClient, tarDoneCh chan<- bool) (*pluginclient.ManifestResponse, error) { generateManifestStream, err := cmpClient.GenerateManifest(ctx, grpc_retry.Disable()) if err != nil { return nil, fmt.Errorf("error getting generateManifestStream: %s", err) @@ -1359,7 +1362,7 @@ func generateManifestsCMP(ctx context.Context, appPath string, env []string, cmp opts := []cmp.SenderOption{ cmp.WithTarDoneChan(tarDoneCh), } - err = cmp.SendApplicationStream(generateManifestStream.Context(), appPath, generateManifestStream, env, opts...) + err = cmp.SendRepoStream(generateManifestStream.Context(), appPath, repoPath, generateManifestStream, env, opts...) if err != nil { return nil, fmt.Errorf("error sending file to cmp-server: %s", err) } diff --git a/test/cmp/plugin.yaml b/test/cmp/plugin.yaml index 0eafc4ecd4520..1184b715d284d 100644 --- a/test/cmp/plugin.yaml +++ b/test/cmp/plugin.yaml @@ -7,6 +7,7 @@ spec: generate: command: [sh, -c, 'kustomize build'] discover: - fileName: "kustomization.yaml" + find: + glob: "**/kustomization.yaml" allowConcurrency: true lockRepo: false diff --git a/util/app/discovery/discovery.go b/util/app/discovery/discovery.go index 2e3d7debfcbe9..8d3f2768ab0a5 100644 --- a/util/app/discovery/discovery.go +++ b/util/app/discovery/discovery.go @@ -29,11 +29,11 @@ func IsManifestGenerationEnabled(sourceType v1alpha1.ApplicationSourceType, enab return enabled } -func Discover(ctx context.Context, root string, enableGenerateManifests map[string]bool) (map[string]string, error) { +func Discover(ctx context.Context, repoPath string, enableGenerateManifests map[string]bool) (map[string]string, error) { apps := make(map[string]string) // Check if it is CMP - conn, _, err := DetectConfigManagementPlugin(ctx, root) + conn, _, err := DetectConfigManagementPlugin(ctx, repoPath) if err == nil { // Found CMP io.Close(conn) @@ -42,14 +42,14 @@ func Discover(ctx context.Context, root string, enableGenerateManifests map[stri return apps, nil } - err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(repoPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { return nil } - dir, err := filepath.Rel(root, filepath.Dir(path)) + dir, err := filepath.Rel(repoPath, filepath.Dir(path)) if err != nil { return err } @@ -85,7 +85,7 @@ func AppType(ctx context.Context, path string, enableGenerateManifests map[strin // 3. check isSupported(path)? // 4.a if no then close connection // 4.b if yes then return conn for detected plugin -func DetectConfigManagementPlugin(ctx context.Context, appPath string) (io.Closer, pluginclient.ConfigManagementPluginServiceClient, error) { +func DetectConfigManagementPlugin(ctx context.Context, repoPath string) (io.Closer, pluginclient.ConfigManagementPluginServiceClient, error) { var conn io.Closer var cmpClient pluginclient.ConfigManagementPluginServiceClient @@ -109,9 +109,9 @@ func DetectConfigManagementPlugin(ctx context.Context, appPath string) (io.Close continue } - isSupported, err := matchRepositoryCMP(ctx, appPath, cmpClient) + isSupported, err := matchRepositoryCMP(ctx, repoPath, cmpClient) if err != nil { - log.Errorf("repository %s is not the match because %v", appPath, err) + log.Errorf("repository %s is not the match because %v", repoPath, err) continue } @@ -126,21 +126,21 @@ func DetectConfigManagementPlugin(ctx context.Context, appPath string) (io.Close } if !connFound { - return nil, nil, fmt.Errorf("Couldn't find cmp-server plugin supporting repository %s", appPath) + return nil, nil, fmt.Errorf("Couldn't find cmp-server plugin supporting repository %s", repoPath) } return conn, cmpClient, err } -// matchRepositoryCMP will send the appPath to the cmp-server. The cmp-server will +// matchRepositoryCMP will send the repoPath to the cmp-server. The cmp-server will // inspect the files and return true if the repo is supported for manifest generation. // Will return false otherwise. -func matchRepositoryCMP(ctx context.Context, appPath string, client pluginclient.ConfigManagementPluginServiceClient) (bool, error) { +func matchRepositoryCMP(ctx context.Context, repoPath string, client pluginclient.ConfigManagementPluginServiceClient) (bool, error) { matchRepoStream, err := client.MatchRepository(ctx, grpc_retry.Disable()) if err != nil { return false, fmt.Errorf("error getting stream client: %s", err) } - err = cmp.SendApplicationStream(ctx, appPath, matchRepoStream, []string{}) + err = cmp.SendRepoStream(ctx, repoPath, repoPath, matchRepoStream, []string{}) if err != nil { return false, fmt.Errorf("error sending stream: %s", err) } diff --git a/util/cmp/stream.go b/util/cmp/stream.go index cd2bf5f6f5ce1..d66aebe8cbf8d 100644 --- a/util/cmp/stream.go +++ b/util/cmp/stream.go @@ -31,10 +31,10 @@ type StreamReceiver interface { Recv() (*pluginclient.AppStreamRequest, error) } -// ReceiveApplicationStream will receive the Application's files and save them -// in destDir. Will return the env entries if no error. Env entries will be nil -// in case of errors. -func ReceiveApplicationStream(ctx context.Context, receiver StreamReceiver, destDir string) ([]*pluginclient.EnvEntry, error) { +// ReceiveRepoStream will receive the repository files and save them +// in destDir. Will return the stream metadata if no error. Metadata +// will be nil in case of errors. +func ReceiveRepoStream(ctx context.Context, receiver StreamReceiver, destDir string) (*pluginclient.ManifestRequestMetadata, error) { header, err := receiver.Recv() if err != nil { return nil, fmt.Errorf("error receiving stream header: %w", err) @@ -56,7 +56,7 @@ func ReceiveApplicationStream(ctx context.Context, receiver StreamReceiver, dest if err != nil { log.Warnf("error removing the tgz file %q: %s", tgzFile.Name(), err) } - return metadata.GetEnv(), nil + return metadata, nil } // SenderOption defines the function type to by used by specific options @@ -94,15 +94,15 @@ func WithTarDoneChan(ch chan<- bool) SenderOption { } } -// SendApplicationStream will compress the files under the given appPath and send +// SendRepoStream will compress the files under the given repoPath and send // them using the plugin stream sender. -func SendApplicationStream(ctx context.Context, appPath string, sender StreamSender, env []string, opts ...SenderOption) error { +func SendRepoStream(ctx context.Context, appPath, repoPath string, sender StreamSender, env []string, opts ...SenderOption) error { opt := newSenderOption(opts...) // compress all files in appPath in tgz - tgz, checksum, err := compressFiles(appPath) + tgz, checksum, err := compressFiles(repoPath) if err != nil { - return fmt.Errorf("error compressing app files: %w", err) + return fmt.Errorf("error compressing repo files: %w", err) } defer closeAndDelete(tgz) if opt.tarDoneChan != nil { @@ -110,8 +110,16 @@ func SendApplicationStream(ctx context.Context, appPath string, sender StreamSen close(opt.tarDoneChan) } + fi, err := tgz.Stat() + if err != nil { + return fmt.Errorf("error getting tgz stat: %w", err) + } + appRelPath, err := files.RelativePath(appPath, repoPath) + if err != nil { + return fmt.Errorf("error building app relative path: %s", err) + } // send metadata first - mr := appMetadataRequest(appPath, env, checksum) + mr := appMetadataRequest(filepath.Base(appPath), appRelPath, env, checksum, fi.Size()) err = sender.Send(mr) if err != nil { return fmt.Errorf("error sending generate manifest metadata to cmp-server: %w", err) @@ -120,7 +128,7 @@ func SendApplicationStream(ctx context.Context, appPath string, sender StreamSen // send the compressed file err = sendFile(ctx, sender, tgz, opt) if err != nil { - return fmt.Errorf("error sending app files to cmp-server: %w", err) + return fmt.Errorf("error sending tgz file to cmp-server: %w", err) } return nil } @@ -263,13 +271,15 @@ func appFileRequest(chunk []byte) *pluginclient.AppStreamRequest { } // appMetadataRequest build the metadata payload for the ManifestRequest -func appMetadataRequest(appPath string, env []string, checksum string) *pluginclient.AppStreamRequest { +func appMetadataRequest(appName, appRelPath string, env []string, checksum string, size int64) *pluginclient.AppStreamRequest { return &pluginclient.AppStreamRequest{ Request: &pluginclient.AppStreamRequest_Metadata{ Metadata: &pluginclient.ManifestRequestMetadata{ - AppName: filepath.Base(appPath), - Env: toEnvEntry(env), - Checksum: checksum, + AppName: appName, + AppRelPath: appRelPath, + Checksum: checksum, + Size_: size, + Env: toEnvEntry(env), }, }, } diff --git a/util/cmp/stream_test.go b/util/cmp/stream_test.go index 33adaa3c8a8e9..18cd8af095cf6 100644 --- a/util/cmp/stream_test.go +++ b/util/cmp/stream_test.go @@ -63,7 +63,7 @@ func TestReceiveApplicationStream(t *testing.T) { go streamMock.sendFile(context.Background(), t, appDir, streamMock, []string{"env1", "env2"}) // when - env, err := cmp.ReceiveApplicationStream(context.Background(), streamMock, workdir) + env, err := cmp.ReceiveRepoStream(context.Background(), streamMock, workdir) // then require.NoError(t, err) @@ -86,7 +86,7 @@ func (m *streamMock) sendFile(ctx context.Context, t *testing.T, basedir string, defer func() { m.done <- true }() - err := cmp.SendApplicationStream(ctx, basedir, sender, env) + err := cmp.SendRepoStream(ctx, basedir, basedir, sender, env) require.NoError(t, err) }