diff --git a/Taskfile.yml b/Taskfile.yml index 3a3ee56a0..e478f3956 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -546,7 +546,6 @@ tasks: server:store:stop: desc: Stop local OCI registry node - internal: true cmds: - docker stop oci-registry diff --git a/api/runtime/v1/discovery_service.pb.go b/api/runtime/v1/discovery_service.pb.go new file mode 100644 index 000000000..f642278c7 --- /dev/null +++ b/api/runtime/v1/discovery_service.pb.go @@ -0,0 +1,159 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.5 +// protoc (unknown) +// source: agntcy/dir/runtime/v1/discovery_service.proto + +package v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ListProcessesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Filters to apply when listing processes. + // Accepts regexp pattern if single value is provided. + Filters []string `protobuf:"bytes,1,rep,name=filters,proto3" json:"filters,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListProcessesRequest) Reset() { + *x = ListProcessesRequest{} + mi := &file_agntcy_dir_runtime_v1_discovery_service_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListProcessesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListProcessesRequest) ProtoMessage() {} + +func (x *ListProcessesRequest) ProtoReflect() protoreflect.Message { + mi := &file_agntcy_dir_runtime_v1_discovery_service_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListProcessesRequest.ProtoReflect.Descriptor instead. +func (*ListProcessesRequest) Descriptor() ([]byte, []int) { + return file_agntcy_dir_runtime_v1_discovery_service_proto_rawDescGZIP(), []int{0} +} + +func (x *ListProcessesRequest) GetFilters() []string { + if x != nil { + return x.Filters + } + return nil +} + +var File_agntcy_dir_runtime_v1_discovery_service_proto protoreflect.FileDescriptor + +var file_agntcy_dir_runtime_v1_discovery_service_proto_rawDesc = string([]byte{ + 0x0a, 0x2d, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x2f, 0x64, 0x69, 0x72, 0x2f, 0x72, 0x75, 0x6e, + 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, + 0x79, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x15, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x2e, 0x64, 0x69, 0x72, 0x2e, 0x72, 0x75, 0x6e, 0x74, + 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x23, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x2f, 0x64, + 0x69, 0x72, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x30, 0x0a, 0x14, 0x4c, + 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x32, 0x72, 0x0a, + 0x10, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x5e, 0x0a, 0x0d, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, + 0x65, 0x73, 0x12, 0x2b, 0x2e, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x2e, 0x64, 0x69, 0x72, 0x2e, + 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, + 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1e, 0x2e, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x2e, 0x64, 0x69, 0x72, 0x2e, 0x72, 0x75, 0x6e, + 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x30, + 0x01, 0x42, 0xcf, 0x01, 0x0a, 0x19, 0x63, 0x6f, 0x6d, 0x2e, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, + 0x2e, 0x64, 0x69, 0x72, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x42, + 0x15, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x2f, 0x64, 0x69, 0x72, 0x2f, + 0x61, 0x70, 0x69, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0xa2, 0x02, + 0x03, 0x41, 0x44, 0x52, 0xaa, 0x02, 0x15, 0x41, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x2e, 0x44, 0x69, + 0x72, 0x2e, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x15, 0x41, + 0x67, 0x6e, 0x74, 0x63, 0x79, 0x5c, 0x44, 0x69, 0x72, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, + 0x65, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x21, 0x41, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x5c, 0x44, 0x69, + 0x72, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x18, 0x41, 0x67, 0x6e, 0x74, 0x63, + 0x79, 0x3a, 0x3a, 0x44, 0x69, 0x72, 0x3a, 0x3a, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x3a, + 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +}) + +var ( + file_agntcy_dir_runtime_v1_discovery_service_proto_rawDescOnce sync.Once + file_agntcy_dir_runtime_v1_discovery_service_proto_rawDescData []byte +) + +func file_agntcy_dir_runtime_v1_discovery_service_proto_rawDescGZIP() []byte { + file_agntcy_dir_runtime_v1_discovery_service_proto_rawDescOnce.Do(func() { + file_agntcy_dir_runtime_v1_discovery_service_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agntcy_dir_runtime_v1_discovery_service_proto_rawDesc), len(file_agntcy_dir_runtime_v1_discovery_service_proto_rawDesc))) + }) + return file_agntcy_dir_runtime_v1_discovery_service_proto_rawDescData +} + +var file_agntcy_dir_runtime_v1_discovery_service_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_agntcy_dir_runtime_v1_discovery_service_proto_goTypes = []any{ + (*ListProcessesRequest)(nil), // 0: agntcy.dir.runtime.v1.ListProcessesRequest + (*Process)(nil), // 1: agntcy.dir.runtime.v1.Process +} +var file_agntcy_dir_runtime_v1_discovery_service_proto_depIdxs = []int32{ + 0, // 0: agntcy.dir.runtime.v1.DiscoveryService.ListProcesses:input_type -> agntcy.dir.runtime.v1.ListProcessesRequest + 1, // 1: agntcy.dir.runtime.v1.DiscoveryService.ListProcesses:output_type -> agntcy.dir.runtime.v1.Process + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_agntcy_dir_runtime_v1_discovery_service_proto_init() } +func file_agntcy_dir_runtime_v1_discovery_service_proto_init() { + if File_agntcy_dir_runtime_v1_discovery_service_proto != nil { + return + } + file_agntcy_dir_runtime_v1_process_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agntcy_dir_runtime_v1_discovery_service_proto_rawDesc), len(file_agntcy_dir_runtime_v1_discovery_service_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_agntcy_dir_runtime_v1_discovery_service_proto_goTypes, + DependencyIndexes: file_agntcy_dir_runtime_v1_discovery_service_proto_depIdxs, + MessageInfos: file_agntcy_dir_runtime_v1_discovery_service_proto_msgTypes, + }.Build() + File_agntcy_dir_runtime_v1_discovery_service_proto = out.File + file_agntcy_dir_runtime_v1_discovery_service_proto_goTypes = nil + file_agntcy_dir_runtime_v1_discovery_service_proto_depIdxs = nil +} diff --git a/api/runtime/v1/discovery_service_grpc.pb.go b/api/runtime/v1/discovery_service_grpc.pb.go new file mode 100644 index 000000000..c52398f0f --- /dev/null +++ b/api/runtime/v1/discovery_service_grpc.pb.go @@ -0,0 +1,151 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc (unknown) +// source: agntcy/dir/runtime/v1/discovery_service.proto + +package v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + DiscoveryService_ListProcesses_FullMethodName = "/agntcy.dir.runtime.v1.DiscoveryService/ListProcesses" +) + +// DiscoveryServiceClient is the client API for DiscoveryService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type DiscoveryServiceClient interface { + // List all record processes based on filters. + ListProcesses(ctx context.Context, in *ListProcessesRequest, opts ...grpc.CallOption) (DiscoveryService_ListProcessesClient, error) +} + +type discoveryServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewDiscoveryServiceClient(cc grpc.ClientConnInterface) DiscoveryServiceClient { + return &discoveryServiceClient{cc} +} + +func (c *discoveryServiceClient) ListProcesses(ctx context.Context, in *ListProcessesRequest, opts ...grpc.CallOption) (DiscoveryService_ListProcessesClient, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &DiscoveryService_ServiceDesc.Streams[0], DiscoveryService_ListProcesses_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &discoveryServiceListProcessesClient{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type DiscoveryService_ListProcessesClient interface { + Recv() (*Process, error) + grpc.ClientStream +} + +type discoveryServiceListProcessesClient struct { + grpc.ClientStream +} + +func (x *discoveryServiceListProcessesClient) Recv() (*Process, error) { + m := new(Process) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// DiscoveryServiceServer is the server API for DiscoveryService service. +// All implementations should embed UnimplementedDiscoveryServiceServer +// for forward compatibility. +type DiscoveryServiceServer interface { + // List all record processes based on filters. + ListProcesses(*ListProcessesRequest, DiscoveryService_ListProcessesServer) error +} + +// UnimplementedDiscoveryServiceServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedDiscoveryServiceServer struct{} + +func (UnimplementedDiscoveryServiceServer) ListProcesses(*ListProcessesRequest, DiscoveryService_ListProcessesServer) error { + return status.Errorf(codes.Unimplemented, "method ListProcesses not implemented") +} +func (UnimplementedDiscoveryServiceServer) testEmbeddedByValue() {} + +// UnsafeDiscoveryServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to DiscoveryServiceServer will +// result in compilation errors. +type UnsafeDiscoveryServiceServer interface { + mustEmbedUnimplementedDiscoveryServiceServer() +} + +func RegisterDiscoveryServiceServer(s grpc.ServiceRegistrar, srv DiscoveryServiceServer) { + // If the following call pancis, it indicates UnimplementedDiscoveryServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&DiscoveryService_ServiceDesc, srv) +} + +func _DiscoveryService_ListProcesses_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ListProcessesRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(DiscoveryServiceServer).ListProcesses(m, &discoveryServiceListProcessesServer{ServerStream: stream}) +} + +type DiscoveryService_ListProcessesServer interface { + Send(*Process) error + grpc.ServerStream +} + +type discoveryServiceListProcessesServer struct { + grpc.ServerStream +} + +func (x *discoveryServiceListProcessesServer) Send(m *Process) error { + return x.ServerStream.SendMsg(m) +} + +// DiscoveryService_ServiceDesc is the grpc.ServiceDesc for DiscoveryService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var DiscoveryService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "agntcy.dir.runtime.v1.DiscoveryService", + HandlerType: (*DiscoveryServiceServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "ListProcesses", + Handler: _DiscoveryService_ListProcesses_Handler, + ServerStreams: true, + }, + }, + Metadata: "agntcy/dir/runtime/v1/discovery_service.proto", +} diff --git a/api/runtime/v1/process.pb.go b/api/runtime/v1/process.pb.go new file mode 100644 index 000000000..a602354f6 --- /dev/null +++ b/api/runtime/v1/process.pb.go @@ -0,0 +1,202 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.5 +// protoc (unknown) +// source: agntcy/dir/runtime/v1/process.proto + +package v1 + +import ( + v1 "github.com/agntcy/dir/api/core/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Information about a agent process instance. +type Process struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Process ID + Pid string `protobuf:"bytes,1,opt,name=pid,proto3" json:"pid,omitempty"` + // Name of the runtime environment + Runtime string `protobuf:"bytes,2,opt,name=runtime,proto3" json:"runtime,omitempty"` + // Process creation timestamp in the RFC3339 format. + CreatedAt string `protobuf:"bytes,3,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + // Process metadata + Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // The record from which this process was created. + Record *v1.RecordMeta `protobuf:"bytes,5,opt,name=record,proto3" json:"record,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Process) Reset() { + *x = Process{} + mi := &file_agntcy_dir_runtime_v1_process_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Process) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Process) ProtoMessage() {} + +func (x *Process) ProtoReflect() protoreflect.Message { + mi := &file_agntcy_dir_runtime_v1_process_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Process.ProtoReflect.Descriptor instead. +func (*Process) Descriptor() ([]byte, []int) { + return file_agntcy_dir_runtime_v1_process_proto_rawDescGZIP(), []int{0} +} + +func (x *Process) GetPid() string { + if x != nil { + return x.Pid + } + return "" +} + +func (x *Process) GetRuntime() string { + if x != nil { + return x.Runtime + } + return "" +} + +func (x *Process) GetCreatedAt() string { + if x != nil { + return x.CreatedAt + } + return "" +} + +func (x *Process) GetAnnotations() map[string]string { + if x != nil { + return x.Annotations + } + return nil +} + +func (x *Process) GetRecord() *v1.RecordMeta { + if x != nil { + return x.Record + } + return nil +} + +var File_agntcy_dir_runtime_v1_process_proto protoreflect.FileDescriptor + +var file_agntcy_dir_runtime_v1_process_proto_rawDesc = string([]byte{ + 0x0a, 0x23, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x2f, 0x64, 0x69, 0x72, 0x2f, 0x72, 0x75, 0x6e, + 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x15, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x2e, 0x64, 0x69, + 0x72, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x1f, 0x61, 0x67, + 0x6e, 0x74, 0x63, 0x79, 0x2f, 0x64, 0x69, 0x72, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, + 0x2f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x9f, 0x02, + 0x0a, 0x07, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x72, + 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x75, + 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, + 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x64, 0x41, 0x74, 0x12, 0x51, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x61, 0x67, 0x6e, 0x74, + 0x63, 0x79, 0x2e, 0x64, 0x69, 0x72, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, + 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x36, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, + 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, + 0x2e, 0x64, 0x69, 0x72, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x63, + 0x6f, 0x72, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x1a, + 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, + 0xc6, 0x01, 0x0a, 0x19, 0x63, 0x6f, 0x6d, 0x2e, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x2e, 0x64, + 0x69, 0x72, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x42, 0x0c, 0x50, + 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x24, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x67, 0x6e, 0x74, 0x63, 0x79, + 0x2f, 0x64, 0x69, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, + 0x2f, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x41, 0x44, 0x52, 0xaa, 0x02, 0x15, 0x41, 0x67, 0x6e, 0x74, + 0x63, 0x79, 0x2e, 0x44, 0x69, 0x72, 0x2e, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x56, + 0x31, 0xca, 0x02, 0x15, 0x41, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x5c, 0x44, 0x69, 0x72, 0x5c, 0x52, + 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x21, 0x41, 0x67, 0x6e, 0x74, + 0x63, 0x79, 0x5c, 0x44, 0x69, 0x72, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, + 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x18, + 0x41, 0x67, 0x6e, 0x74, 0x63, 0x79, 0x3a, 0x3a, 0x44, 0x69, 0x72, 0x3a, 0x3a, 0x52, 0x75, 0x6e, + 0x74, 0x69, 0x6d, 0x65, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +}) + +var ( + file_agntcy_dir_runtime_v1_process_proto_rawDescOnce sync.Once + file_agntcy_dir_runtime_v1_process_proto_rawDescData []byte +) + +func file_agntcy_dir_runtime_v1_process_proto_rawDescGZIP() []byte { + file_agntcy_dir_runtime_v1_process_proto_rawDescOnce.Do(func() { + file_agntcy_dir_runtime_v1_process_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agntcy_dir_runtime_v1_process_proto_rawDesc), len(file_agntcy_dir_runtime_v1_process_proto_rawDesc))) + }) + return file_agntcy_dir_runtime_v1_process_proto_rawDescData +} + +var file_agntcy_dir_runtime_v1_process_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_agntcy_dir_runtime_v1_process_proto_goTypes = []any{ + (*Process)(nil), // 0: agntcy.dir.runtime.v1.Process + nil, // 1: agntcy.dir.runtime.v1.Process.AnnotationsEntry + (*v1.RecordMeta)(nil), // 2: agntcy.dir.core.v1.RecordMeta +} +var file_agntcy_dir_runtime_v1_process_proto_depIdxs = []int32{ + 1, // 0: agntcy.dir.runtime.v1.Process.annotations:type_name -> agntcy.dir.runtime.v1.Process.AnnotationsEntry + 2, // 1: agntcy.dir.runtime.v1.Process.record:type_name -> agntcy.dir.core.v1.RecordMeta + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_agntcy_dir_runtime_v1_process_proto_init() } +func file_agntcy_dir_runtime_v1_process_proto_init() { + if File_agntcy_dir_runtime_v1_process_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agntcy_dir_runtime_v1_process_proto_rawDesc), len(file_agntcy_dir_runtime_v1_process_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_agntcy_dir_runtime_v1_process_proto_goTypes, + DependencyIndexes: file_agntcy_dir_runtime_v1_process_proto_depIdxs, + MessageInfos: file_agntcy_dir_runtime_v1_process_proto_msgTypes, + }.Build() + File_agntcy_dir_runtime_v1_process_proto = out.File + file_agntcy_dir_runtime_v1_process_proto_goTypes = nil + file_agntcy_dir_runtime_v1_process_proto_depIdxs = nil +} diff --git a/discovery/README.md b/discovery/README.md new file mode 100644 index 000000000..e750e5455 --- /dev/null +++ b/discovery/README.md @@ -0,0 +1,262 @@ +# Service Discovery + +Network-aware service discovery for runtime workloads. Watches processes in a runtimes (Docker, containerd, Kubernetes) and exposes an HTTP API for discovering reachable services based on network isolation. + +## Architecture + +``` + ┌─────────────────────────────────────┐ + │ etcd │ + │ (distributed metadata store) │ + │ │ + │ /discovery/workloads/{id} ◄─── watcher writes + │ /discovery/metadata/{id}/{proc} ◄── inspector writes + └──────────┬──────────────┬───────────┘ + │ │ + write │ │ read (watch) + │ │ + ┌────────────────────┴───┐ ┌─────┴────────────────────┐ + │ Watcher │ │ Server │ + │ - Watches runtime │ │ - HTTP API │ + │ - Tracks networks │ │ - Reachability queries │ + │ - Tracks workloads │ │ - Filtering by network │ + └────────────┬───────────┘ └──────────────────────────┘ + │ + │ watch ┌──────────────────────────┐ + │ │ Inspector │ + ┌──────────────────┼──────────────────│ - Watches workloads │ + │ │ │ - Health checks │ + │ │ │ - OpenAPI discovery │ + ┌──────┴──────┐ ┌──────┴──────┐ ┌──────┴─────┐────────────────────┘ + │ Docker │ │ containerd │ │ Kubernetes │ + │ Socket │ │ Socket │ │ API │ + └──────┬──────┘ └──────┬──────┘ └──────┬─────┘ + │ │ │ + └──────────────────┼──────────────────┘ + │ + ┌──────────────────┼──────────────────┐ + │ │ │ + ┌────┴────┐ ┌────┴────┐ ┌────┴────┐ + │ team-a │ │ team-b │ │ * │ + ├─────────┤ ├─────────┤ ├─────────┤ + │service-1│ │service-2│ │service-5│ + │service-3│ │service-4│ │ │ + └─────────┘ └─────────┘ └─────────┘ +``` + +### Key Structure + +| Prefix | Owner | Description | +|--------|-------|-------------| +| `/discovery/workloads/{id}` | Watcher | Workload JSON (container/pod info, networks, ports) | +| `/discovery/metadata/{id}/{processor}` | Inspector | Processor metadata (health, openapi, etc.) | + +**Benefits of separate prefixes:** +- **Clean watches**: Inspector watches only workloads, ignores metadata changes +- **Clear ownership**: Watcher owns workloads, Inspector owns metadata +- **Efficient**: No key parsing needed to filter events + +**Components:** + +| Component | Path | Description | +| ----------- | ------------ | -------------------------------------------------- | +| `watcher` | `./watcher` | Watches runtime (Docker/containerd/K8s) for workloads and writes to etcd | +| `server` | `./server` | HTTP API for querying discovered services and reachability | +| `inspector` | `./inspector`| Watches etcd for workloads and extracts metadata (health, OpenAPI) | + +## Quick Start + +### Docker Compose + +```bash +cd discovery +docker compose up -d --build + +# Check stats +curl http://localhost:8080/stats + +# Discover services (from hostname or name) +CID=$(docker ps -q -f name=service-1) +curl http://localhost:8080/discover?from=$CID | jq . + +# Cleanup +docker compose down +``` + +### Docker Swarm + +```bash +cd discovery +docker swarm init +docker build -t discovery-watcher:latest ./watcher +docker stack deploy -c docker-compose.swarm.yml discovery + +# Wait for services to start +sleep 10 + +# Check stats +curl http://localhost:8080/stats + +# Discover services (from hostname or name) +CID=$(docker ps -q -f name=discovery_service-1) +curl http://localhost:8080/discover?from=$CID | jq . + +# Cleanup +docker stack rm discovery +docker swarm leave --force +``` + +### containerd (Linux or Lima VM) + +containerd requires direct socket access, which isn't available on macOS. Use Lima to create a Linux VM. +For network isolation, ensure that CNI is set up and that containerd is using CNI networks. + +```bash +# Setup Lima for containerd (macOS) +brew install lima +limactl create --name=discovery +limactl start discovery +limactl shell discovery # in a new terminal + +# Inside VM - start watcher +cd discovery +nerdctl compose -f docker-compose.containerd.yml up -d + +# Check stats +curl http://localhost:8080/stats + +# Discover services (from hostname or name) +CID=$(nerdctl ps -q -f name=service-1) +curl http://localhost:8080/discover?from=$CID | jq . + +# Cleanup +nerdctl compose down + +# Exit VM and cleanup (for macOS) +exit +limactl stop discovery +limactl delete discovery +``` + +### Kubernetes (kind/minikube) + +Test inside a local Kubernetes cluster using kind or minikube. + +```bash +cd discovery + +# Create cluster (kind) +kind create cluster --name discovery-test + +# Build and load images +docker build -t discovery-watcher:latest ./watcher +docker build -t discovery-server:latest ./server +docker build -t discovery-inspector:latest ./inspector +kind load docker-image discovery-watcher:latest --name discovery-test +kind load docker-image discovery-server:latest --name discovery-test +kind load docker-image discovery-inspector:latest --name discovery-test + +# Deploy everything (etcd + watcher + server + inspector + test workloads) +kubectl apply -f k8s.discovery.yaml +kubectl wait --for=condition=ready pod -l app=discovery-watcher --timeout=60s +kubectl port-forward svc/discovery-server 8080:8080 # in a new terminal + +# Check stats +curl http://localhost:8080/stats | jq . + +# Discover services (from hostname or name) +PID=$(kubectl get pod service-1 -n team-a -o jsonpath='{.metadata.uid}') +curl "http://localhost:8080/discover?from=$PID" | jq . + +# Check inspector logs +kubectl logs -l app=discovery-inspector --follow + +# Cleanup +kind delete cluster --name discovery-test +``` + +## API + +| Endpoint | Description | +| ------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `GET /discover?from={id}` | Discover workloads reachable from source. Hostnames or full container IDs are accepted for source identification. | +| `GET /workloads` | List all registered workloads | +| `GET /stats` | Registry statistics | +| `GET /health` | Health check | + +## Configuration + +### Web Server + +| Variable | Default | Description | +| ------------- | --------- | ------------------------ | +| `SERVER_HOST` | `0.0.0.0` | HTTP server bind address | +| `SERVER_PORT` | `8080` | HTTP server port | +| `DEBUG` | `false` | Enable debug mode | + +### Storage + +| Variable | Default | Description | +| --------------- | ------------ | ---------------------------- | +| `ETCD_HOST` | `localhost` | etcd hostname | +| `ETCD_PORT` | `2379` | etcd port | +| `ETCD_PREFIX` | `/discovery` | Key prefix for etcd storage | + +### Runtime + +| Variable | Default | Description | +| --------------------------- | --------------------------------- | ------------------------------------------------------- | +| `RUNTIME` | `docker` | Runtime to watch (`docker`, `containerd`, `kubernetes`) | +| `DOCKER_SOCKET` | `unix:///var/run/docker.sock` | Docker socket path | +| `DOCKER_LABEL_KEY` | `discover` | Label key for discoverable containers | +| `DOCKER_LABEL_VALUE` | `true` | Label value to match | +| `CONTAINERD_SOCKET` | `/run/containerd/containerd.sock` | containerd socket path | +| `CONTAINERD_NAMESPACE` | `default` | containerd namespace to watch | +| `CONTAINERD_CNI_STATE_DIR` | `/var/lib/cni/results` | CNI state directory for network info | +| `CONTAINERD_LABEL_KEY` | `discover` | Label key for discoverable containers | +| `CONTAINERD_LABEL_VALUE` | `true` | Label value to match | +| `KUBECONFIG` | - | Path to kubeconfig file | +| `KUBERNETES_NAMESPACE` | - | Namespace to watch (all if not set) | +| `KUBERNETES_IN_CLUSTER` | `false` | Use in-cluster config | +| `KUBERNETES_LABEL_KEY` | `discover` | Label key for discoverable pods | +| `KUBERNETES_LABEL_VALUE` | `true` | Label value to match | +| `KUBERNETES_WATCH_SERVICES` | `true` | Watch services in addition to pods | + +### Inspector + +| Variable | Default | Description | +| ----------------------- | ------------------------ | ---------------------------------------------- | +| `ETCD_PREFIX` | `/discovery/workloads/` | etcd prefix for workloads and metadata | +| `HEALTH_ENABLED` | `true` | Enable health check processor | +| `HEALTH_TIMEOUT` | `5` | Health check timeout (seconds) | +| `HEALTH_PATHS` | `/` | Paths to probe for health | +| `OPENAPI_ENABLED` | `true` | Enable OpenAPI discovery processor | +| `OPENAPI_TIMEOUT` | `10` | OpenAPI fetch timeout (seconds) | +| `OPENAPI_PATHS` | `/openapi.json,/swagger.json,/api-docs` | Paths to check for OpenAPI spec | +| `PROCESSOR_WORKERS` | `4` | Number of worker threads | +| `PROCESSOR_RETRY_COUNT` | `3` | Number of retries for failed processing | +| `PROCESSOR_RETRY_DELAY` | `5` | Delay between retries (seconds) | + +## Network Isolation + +Services can only discover other services on shared networks. + +| Service | Networks | Can Discover | +| --------- | -------------- | ------------------------------------------ | +| service-1 | team-a | service-3, service-5 | +| service-2 | team-b | service-4, service-5 | +| service-5 | team-a, team-b | service-1, service-2, service-3, service-4 | + +## Labeling Services + +Add the `discover=true` label to make a service discoverable: + +```yaml +services: + my-service: + image: nginx:alpine + labels: + - "discover=true" + networks: + - my-network +``` diff --git a/discovery/docker-compose.containerd.yml b/discovery/docker-compose.containerd.yml new file mode 100644 index 000000000..23a436b9c --- /dev/null +++ b/discovery/docker-compose.containerd.yml @@ -0,0 +1,177 @@ +# Docker Compose for testing containerd runtime +# +# Requirements: +# - Linux host with containerd installed (not via Docker) +# - containerd socket at /run/containerd/containerd.sock +# - CNI configured with state at /var/lib/cni/results +# +# Usage: +# nerdctl compose -f docker-compose.containerd.yml up -d +# +# Note: This is for testing on a Linux machine with standalone containerd. +# On macOS/Windows, containerd can be setup with Lima. +# +services: + # ============================================================ + # etcd - Distributed key-value store + # ============================================================ + etcd: + image: quay.io/coreos/etcd:v3.5.17 + container_name: etcd + command: + - etcd + - --name=etcd0 + - --data-dir=/etcd-data + - --advertise-client-urls=http://etcd:2379 + - --listen-client-urls=http://0.0.0.0:2379 + - --initial-advertise-peer-urls=http://etcd:2380 + - --listen-peer-urls=http://0.0.0.0:2380 + - --initial-cluster=etcd0=http://etcd:2380 + networks: + - discovery + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 5s + timeout: 3s + retries: 5 + + # ============================================================ + # Watcher - containerd runtime + # ============================================================ + watcher: + build: ./watcher + volumes: + # containerd socket (Linux only) + - /run/containerd/containerd.sock:/run/containerd/containerd.sock:ro + # CNI state directory for network info + - /var/lib/cni/results:/var/lib/cni/results:ro + environment: + # etcd + - ETCD_HOST=etcd + - ETCD_PORT=2379 + - ETCD_WORKLOADS_PREFIX=/discovery/workloads/ + # Runtime + - RUNTIME=containerd + - CONTAINERD_SOCKET=/run/containerd/containerd.sock + - CONTAINERD_NAMESPACE=default + - CONTAINERD_CNI_STATE_DIR=/var/lib/cni/results + - CONTAINERD_LABEL_KEY=discover + - CONTAINERD_LABEL_VALUE=true + - PYTHONUNBUFFERED=1 + depends_on: + etcd: + condition: service_healthy + networks: + - discovery + + # ============================================================ + # Server - HTTP API for querying discovered services + # ============================================================ + server: + build: ./server + environment: + # etcd + - ETCD_HOST=etcd + - ETCD_PORT=2379 + - ETCD_WORKLOADS_PREFIX=/discovery/workloads/ + - ETCD_METADATA_PREFIX=/discovery/metadata/ + # Server + - SERVER_HOST=0.0.0.0 + - SERVER_PORT=8080 + - PYTHONUNBUFFERED=1 + depends_on: + etcd: + condition: service_healthy + networks: + - discovery + ports: + - "8080:8080" + + # ============================================================ + # Inspector - Watches workloads and extracts metadata + # ============================================================ + inspector: + build: ./inspector + environment: + # etcd + - ETCD_HOST=etcd + - ETCD_PORT=2379 + - ETCD_WORKLOADS_PREFIX=/discovery/workloads/ + - ETCD_METADATA_PREFIX=/discovery/metadata/ + # Processors + - HEALTH_ENABLED=true + - HEALTH_TIMEOUT=5 + - HEALTH_PATHS=/ + - OPENAPI_ENABLED=true + - OPENAPI_TIMEOUT=10 + - OPENAPI_PATHS=/openapi.json,/swagger.json,/api-docs + - PROCESSOR_WORKERS=4 + - PYTHONUNBUFFERED=1 + depends_on: + etcd: + condition: service_healthy + watcher: + condition: service_started + networks: + - discovery + - team-a + - team-b + + # ============================================================ + # Example services - must have discover=true label + # ============================================================ + service-1: + image: nginx:alpine + labels: + - "discover=true" + networks: + - team-a + + service-2: + image: nginx:alpine + labels: + - "discover=true" + networks: + - team-b + + service-3: + image: nginx:alpine + labels: + - "discover=true" + networks: + - team-a + + service-4: + image: nginx:alpine + labels: + - "discover=true" + networks: + - team-b + + service-5: + image: nginx:alpine + labels: + - "discover=true" + networks: + - team-a + - team-b + + # Service without discover label (NOT tracked) + service-private: + image: nginx:alpine + container_name: service-private + networks: + - team-internal + +networks: + discovery: + driver: bridge + + team-a: + driver: bridge + + team-b: + driver: bridge + + team-internal: + driver: bridge diff --git a/discovery/docker-compose.swarm.yml b/discovery/docker-compose.swarm.yml new file mode 100644 index 000000000..fd9e2bc17 --- /dev/null +++ b/discovery/docker-compose.swarm.yml @@ -0,0 +1,195 @@ +# Docker Swarm Stack for Service Discovery +# +# Deploy with: +# 1. Build images: docker compose build watcher server +# 2. Init swarm (if needed): docker swarm init +# 3. Deploy: docker stack deploy -c docker-compose.swarm.yml discovery +# +# Architecture: +# - etcd runs as single replica (use 3 for production HA) +# - watcher runs as global service (one per node) - watches Docker, writes to etcd +# - server runs as replicated service - reads from etcd, serves HTTP API +# +services: + # ============================================================ + # etcd - key-value store for service discovery + # ============================================================ + etcd: + image: quay.io/coreos/etcd:v3.5.17 + command: + - etcd + - --name=etcd0 + - --data-dir=/etcd-data + - --advertise-client-urls=http://etcd:2379 + - --listen-client-urls=http://0.0.0.0:2379 + - --initial-advertise-peer-urls=http://etcd:2380 + - --listen-peer-urls=http://0.0.0.0:2380 + - --initial-cluster=etcd0=http://etcd:2380 + networks: + - discovery + deploy: + replicas: 1 + placement: + constraints: + - node.role == manager + restart_policy: + condition: on-failure + delay: 5s + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 10s + timeout: 5s + retries: 3 + + # ============================================================ + # Watcher - Global service (one per node) - watches Docker socket + # ============================================================ + watcher: + image: discovery-watcher:latest + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + # etcd + - ETCD_HOST=etcd + - ETCD_PORT=2379 + - ETCD_WORKLOADS_PREFIX=/discovery/workloads/ + # Runtime + - RUNTIME=docker + - PYTHONUNBUFFERED=1 + networks: + - discovery + deploy: + mode: global + restart_policy: + condition: on-failure + delay: 5s + + # ============================================================ + # Server - HTTP API for querying discovered services + # ============================================================ + server: + image: discovery-server:latest + environment: + # etcd + - ETCD_HOST=etcd + - ETCD_PORT=2379 + - ETCD_WORKLOADS_PREFIX=/discovery/workloads/ + - ETCD_METADATA_PREFIX=/discovery/metadata/ + # Server + - SERVER_HOST=0.0.0.0 + - SERVER_PORT=8080 + - PYTHONUNBUFFERED=1 + networks: + - discovery + ports: + - "8080:8080" + deploy: + replicas: 2 + restart_policy: + condition: on-failure + delay: 5s + + # ============================================================ + # Inspector - Watches workloads and extracts metadata + # ============================================================ + inspector: + image: discovery-inspector:latest + environment: + # etcd + - ETCD_HOST=etcd + - ETCD_PORT=2379 + - ETCD_WORKLOADS_PREFIX=/discovery/workloads/ + - ETCD_METADATA_PREFIX=/discovery/metadata/ + # Processors + - HEALTH_ENABLED=true + - HEALTH_TIMEOUT=5 + - HEALTH_PATHS=/ + - OPENAPI_ENABLED=true + - OPENAPI_TIMEOUT=10 + - OPENAPI_PATHS=/openapi.json,/swagger.json,/api-docs + - PROCESSOR_WORKERS=4 + - PYTHONUNBUFFERED=1 + networks: + - discovery + - team-a + - team-b + deploy: + replicas: 1 + restart_policy: + condition: on-failure + delay: 5s + + # ============================================================ + # Example services - labels go at service level for container labels + # ============================================================ + service-1: + image: nginx:alpine + labels: + discover: "true" + networks: + - team-a + deploy: + replicas: 1 + + service-2: + image: nginx:alpine + labels: + discover: "true" + networks: + - team-b + deploy: + replicas: 1 + + service-3: + image: nginx:alpine + labels: + discover: "true" + networks: + - team-a + deploy: + replicas: 1 + + service-4: + image: nginx:alpine + labels: + discover: "true" + networks: + - team-b + deploy: + replicas: 1 + + service-5: + image: nginx:alpine + labels: + discover: "true" + networks: + - team-a + - team-b + deploy: + replicas: 1 + + # No discover label - will NOT be tracked + service-private: + image: nginx:alpine + networks: + - team-internal + deploy: + replicas: 1 + +networks: + discovery: + driver: overlay + attachable: true + + team-a: + driver: overlay + attachable: true + + team-b: + driver: overlay + attachable: true + + team-internal: + driver: overlay + attachable: true + diff --git a/discovery/docker-compose.yml b/discovery/docker-compose.yml new file mode 100644 index 000000000..00d14d00e --- /dev/null +++ b/discovery/docker-compose.yml @@ -0,0 +1,160 @@ +services: + # ============================================================ + # etcd - Distributed key-value store for service discovery data + # ============================================================ + etcd: + image: quay.io/coreos/etcd:v3.5.17 + container_name: etcd + command: + - etcd + - --name=etcd0 + - --data-dir=/etcd-data + - --advertise-client-urls=http://etcd:2379 + - --listen-client-urls=http://0.0.0.0:2379 + - --initial-advertise-peer-urls=http://etcd:2380 + - --listen-peer-urls=http://0.0.0.0:2380 + - --initial-cluster=etcd0=http://etcd:2380 + networks: + - discovery + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 5s + timeout: 3s + retries: 5 + + # ============================================================ + # Watcher - Watches container runtime and syncs to etcd + # ============================================================ + watcher: + build: ./watcher + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + # etcd + - ETCD_HOST=etcd + - ETCD_PORT=2379 + - ETCD_WORKLOADS_PREFIX=/discovery/workloads/ + # Runtime (docker, containerd, or kubernetes) + - RUNTIME=docker + - PYTHONUNBUFFERED=1 + depends_on: + etcd: + condition: service_healthy + networks: + - discovery + + # ============================================================ + # Server - HTTP API for querying discovered services + # ============================================================ + server: + build: ./server + environment: + # etcd + - ETCD_HOST=etcd + - ETCD_PORT=2379 + - ETCD_WORKLOADS_PREFIX=/discovery/workloads/ + - ETCD_METADATA_PREFIX=/discovery/metadata/ + # Server + - SERVER_HOST=0.0.0.0 + - SERVER_PORT=8080 + - PYTHONUNBUFFERED=1 + depends_on: + etcd: + condition: service_healthy + networks: + - discovery + ports: + - "8080:8080" + + # ============================================================ + # Inspector - Watches workloads and extracts metadata + # ============================================================ + inspector: + build: ./inspector + environment: + # etcd + - ETCD_HOST=etcd + - ETCD_PORT=2379 + - ETCD_WORKLOADS_PREFIX=/discovery/workloads/ + - ETCD_METADATA_PREFIX=/discovery/metadata/ + # Processors + - HEALTH_ENABLED=true + - HEALTH_TIMEOUT=5 + - HEALTH_PATHS=/ + - OPENAPI_ENABLED=true + - OPENAPI_TIMEOUT=10 + - OPENAPI_PATHS=/openapi.json,/swagger.json,/api-docs + - PROCESSOR_WORKERS=4 + - PYTHONUNBUFFERED=1 + depends_on: + etcd: + condition: service_healthy + watcher: + condition: service_started + networks: + - discovery + - team-a + - team-b + + # ============================================================ + # Example services - must be on discovery network to be tracked + # ============================================================ + + + service-1: + image: nginx:alpine + labels: + - "discover=true" + networks: + - team-a + + service-2: + image: nginx:alpine + labels: + - "discover=true" + networks: + - team-b + + # Service Internal - on team-a-internal, will be auto-connected to discovery + service-3: + image: nginx:alpine + labels: + - "discover=true" + networks: + - team-a + + service-4: + image: nginx:alpine + labels: + - "discover=true" + networks: + - team-b + + service-5: + image: nginx:alpine + labels: + - "discover=true" + networks: + - team-a + - team-b + + # Service Private - no discover label (NOT tracked, NOT auto-connected) + service-private: + image: nginx:alpine + container_name: service-private + networks: + - team-internal + +networks: + # Discovery network - watcher lives here, services auto-connected + discovery: + driver: bridge + + team-a: + driver: bridge + + team-b: + driver: bridge + + team-internal: + driver: bridge diff --git a/discovery/inspector/Dockerfile b/discovery/inspector/Dockerfile new file mode 100644 index 000000000..4686dbd4a --- /dev/null +++ b/discovery/inspector/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD ["python", "main.py"] diff --git a/discovery/inspector/config.py b/discovery/inspector/config.py new file mode 100644 index 000000000..68cb83a53 --- /dev/null +++ b/discovery/inspector/config.py @@ -0,0 +1,71 @@ +""" +Configuration for the metadata inspector service. +""" + +import logging +import os +import sys +from dataclasses import dataclass, field +from typing import Optional + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - [%(name)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + stream=sys.stdout, + force=True +) + +logger = logging.getLogger("discovery.inspector") + + +@dataclass +class EtcdConfig: + """etcd storage configuration.""" + host: str = field(default_factory=lambda: os.getenv("ETCD_HOST", "localhost")) + port: int = field(default_factory=lambda: int(os.getenv("ETCD_PORT", "2379"))) + workloads_prefix: str = field(default_factory=lambda: os.getenv("ETCD_WORKLOADS_PREFIX", "/discovery/workloads/")) + metadata_prefix: str = field(default_factory=lambda: os.getenv("ETCD_METADATA_PREFIX", "/discovery/metadata/")) + username: Optional[str] = field(default_factory=lambda: os.getenv("ETCD_USERNAME")) + password: Optional[str] = field(default_factory=lambda: os.getenv("ETCD_PASSWORD")) + + @property + def url(self) -> str: + return f"http://{self.host}:{self.port}" + + +@dataclass +class ProcessorConfig: + """Processor configuration.""" + # Health check settings + health_enabled: bool = field(default_factory=lambda: os.getenv("HEALTH_ENABLED", "true").lower() == "true") + health_timeout: int = field(default_factory=lambda: int(os.getenv("HEALTH_TIMEOUT", "5"))) + health_paths: list = field(default_factory=lambda: os.getenv("HEALTH_PATHS", "/").split(",")) + + # OpenAPI settings + openapi_enabled: bool = field(default_factory=lambda: os.getenv("OPENAPI_ENABLED", "true").lower() == "true") + openapi_timeout: int = field(default_factory=lambda: int(os.getenv("OPENAPI_TIMEOUT", "10"))) + openapi_paths: list = field(default_factory=lambda: os.getenv("OPENAPI_PATHS", "/openapi.json,/swagger.json,/api-docs").split(",")) + + # Processing settings + retry_count: int = field(default_factory=lambda: int(os.getenv("PROCESSOR_RETRY_COUNT", "3"))) + retry_delay: int = field(default_factory=lambda: int(os.getenv("PROCESSOR_RETRY_DELAY", "5"))) + worker_count: int = field(default_factory=lambda: int(os.getenv("PROCESSOR_WORKERS", "4"))) + + +@dataclass +class Config: + """Main configuration for the inspector.""" + etcd: EtcdConfig = field(default_factory=EtcdConfig) + processor: ProcessorConfig = field(default_factory=ProcessorConfig) + + @classmethod + def from_env(cls) -> "Config": + """Create configuration from environment variables.""" + return cls() + + +def load_config() -> Config: + """Load configuration from environment.""" + return Config.from_env() diff --git a/discovery/inspector/main.py b/discovery/inspector/main.py new file mode 100644 index 000000000..78d0f45a3 --- /dev/null +++ b/discovery/inspector/main.py @@ -0,0 +1,191 @@ +""" +Metadata Inspector entry point. + +Watches for workloads in etcd and runs processors to extract metadata. +""" + +import signal +import sys +import threading +import queue +from typing import Optional + +from models import Workload +from config import load_config, Config, logger +from storage import create_storage, StorageInterface +from processors import create_processors, ProcessorInterface + + +class MetadataInspector: + """ + Metadata inspector coordinator. + + Watches etcd for workload changes and runs processors to extract metadata. + """ + + def __init__(self, config: Config): + self.config = config + self.storage: Optional[StorageInterface] = None + self.processors: list[ProcessorInterface] = [] + self._running = False + self._stop_event = threading.Event() + self._work_queue: queue.Queue = queue.Queue() + self._workers: list[threading.Thread] = [] + + def _process_workload(self, workload: Workload) -> None: + """Run all processors on a workload.""" + for processor in self.processors: + if not processor.should_process(workload): + continue + + try: + result = processor.process(workload) + if result: + self.storage.set_metadata(workload.id, processor.name, result) + except Exception as e: + logger.error( + "Processor %s failed for %s: %s", + processor.name, + workload.name, + e + ) + + def _worker(self, worker_id: int) -> None: + """Worker thread that processes workloads from the queue.""" + logger.info("Worker %d started", worker_id) + + while not self._stop_event.is_set(): + try: + # Get workload from queue with timeout + try: + event_type, workload = self._work_queue.get(timeout=1) + except queue.Empty: + continue + + if event_type == "PUT" and workload: + logger.debug( + "Worker %d processing %s (%s)", + worker_id, + workload.name, + workload.id[:12] + ) + self._process_workload(workload) + + elif event_type == "DELETE" and workload: + # Clean up metadata when workload is deleted + self.storage.delete_metadata(workload.id) + + self._work_queue.task_done() + + except Exception as e: + logger.error("Worker %d error: %s", worker_id, e) + + logger.info("Worker %d stopped", worker_id) + + def _watcher(self) -> None: + """Watch etcd for workload changes and queue them.""" + logger.info("Watcher started") + + try: + for event_type, workload in self.storage.watch_workloads(): + if self._stop_event.is_set(): + break + + if workload: + logger.info( + "[%s] Queued %s (%s)", + event_type, + workload.name, + workload.id[:12] + ) + self._work_queue.put((event_type, workload)) + + except Exception as e: + logger.error("Watcher error: %s", e) + + logger.info("Watcher stopped") + + def start(self) -> None: + """Start the metadata inspector.""" + self._running = True + + # Initialize storage + self.storage = create_storage(self.config) + + # Initialize processors + self.processors = create_processors(self.config) + if not self.processors: + logger.error("Failed to initialize processors!") + sys.exit(1) + + # Start worker threads + worker_count = self.config.processor.worker_count + for i in range(worker_count): + worker = threading.Thread( + target=self._worker, + args=(i,), + daemon=True, + name=f"worker-{i}" + ) + worker.start() + self._workers.append(worker) + + logger.info("Started %d worker threads", worker_count) + + # Start watcher in main thread + logger.info("Inspector running, press Ctrl+C to stop...") + self._watcher() + + def stop(self) -> None: + """Stop the metadata inspector.""" + logger.info("Stopping inspector...") + self._running = False + self._stop_event.set() + + # Wait for workers to finish + for worker in self._workers: + worker.join(timeout=5) + + # Close storage + if self.storage: + self.storage.close() + + logger.info("Inspector stopped") + + +def main(): + """Main entry point.""" + config = load_config() + + # Log configuration + logger.info("=" * 60) + logger.info("Metadata Inspector") + logger.info("=" * 60) + logger.info("etcd: %s:%d", config.etcd.host, config.etcd.port) + logger.info("Workloads prefix: %s", config.etcd.workloads_prefix) + logger.info("Metadata prefix: %s", config.etcd.metadata_prefix) + logger.info("Workers: %d", config.processor.worker_count) + logger.info("Health check: %s", "enabled" if config.processor.health_enabled else "disabled") + logger.info("OpenAPI discovery: %s", "enabled" if config.processor.openapi_enabled else "disabled") + logger.info("=" * 60) + + inspector = MetadataInspector(config) + + # Handle signals + def signal_handler(sig, frame): + logger.info("Received signal %s, shutting down...", sig) + inspector.stop() + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Start + try: + inspector.start() + except KeyboardInterrupt: + inspector.stop() + + +if __name__ == "__main__": + main() diff --git a/discovery/inspector/models.py b/discovery/inspector/models.py new file mode 100644 index 000000000..0ae0b07d5 --- /dev/null +++ b/discovery/inspector/models.py @@ -0,0 +1,50 @@ +""" +Data models for the inspector service. +""" + +import json +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class Workload: + """Workload data from runtime storage.""" + id: str + name: str + hostname: str + runtime: str + workload_type: str + addresses: list = field(default_factory=list) + isolation_groups: list = field(default_factory=list) + ports: list = field(default_factory=list) + labels: dict = field(default_factory=dict) + annotations: dict = field(default_factory=dict) + node: Optional[str] = None + namespace: Optional[str] = None + metadata: dict = field(default_factory=dict) + registrar: Optional[str] = None + + @classmethod + def from_dict(cls, data: dict) -> "Workload": + return cls( + id=data.get("id", ""), + name=data.get("name", ""), + hostname=data.get("hostname", ""), + runtime=data.get("runtime", ""), + workload_type=data.get("workload_type", data.get("type", "")), + addresses=data.get("addresses", []), + isolation_groups=data.get("isolation_groups", []), + ports=data.get("ports", []), + labels=data.get("labels", {}), + annotations=data.get("annotations", {}), + node=data.get("node"), + namespace=data.get("namespace"), + metadata=data.get("metadata", {}), + registrar=data.get("registrar"), + ) + + @classmethod + def from_json(cls, data: str) -> "Workload": + return cls.from_dict(json.loads(data)) + diff --git a/discovery/inspector/processors/__init__.py b/discovery/inspector/processors/__init__.py new file mode 100644 index 000000000..6f8efed42 --- /dev/null +++ b/discovery/inspector/processors/__init__.py @@ -0,0 +1,29 @@ +""" +Processors module for inspector. +""" + +from typing import List + +from processors.interface import ProcessorInterface +from processors.health import HealthProcessor +from processors.openapi import OpenAPIProcessor +from config import logger, Config + + +def create_processors(config: Config) -> List[ProcessorInterface]: + """Initialize enabled processors.""" + processors: List[ProcessorInterface] = [] + + # Health check processor + health = HealthProcessor(config.processor) + if health.enabled: + processors.append(health) + logger.info("Enabled processor: %s", health.name) + + # OpenAPI processor + openapi = OpenAPIProcessor(config.processor) + if openapi.enabled: + processors.append(openapi) + logger.info("Enabled processor: %s", openapi.name) + + return processors diff --git a/discovery/inspector/processors/health.py b/discovery/inspector/processors/health.py new file mode 100644 index 000000000..169755d0e --- /dev/null +++ b/discovery/inspector/processors/health.py @@ -0,0 +1,158 @@ +""" +Health check processor. + +Probes workloads for health endpoints and records results. +""" + +import json +import time +from dataclasses import dataclass, field, asdict +from datetime import datetime +from typing import Optional + +import requests + +from config import ProcessorConfig, logger +from models import Workload +from processors.interface import ProcessorInterface + + +# ==================== Result Model ==================== + +@dataclass +class HealthResult: + """Health check result.""" + healthy: bool + endpoint: Optional[str] = None + status_code: Optional[int] = None + response_time_ms: Optional[float] = None + error: Optional[str] = None + checked_at: str = field(default_factory=lambda: datetime.utcnow().isoformat()) + + def to_dict(self) -> dict: + return asdict(self) + + def to_json(self) -> str: + return json.dumps(self.to_dict()) + + +# ==================== Processor ==================== + +class HealthProcessor(ProcessorInterface): + """ + Health check processor. + + Probes common health endpoints on workloads and records: + - Whether the workload is healthy + - Which endpoint responded + - Response time + """ + + def __init__(self, config: ProcessorConfig): + self.config = config + self.timeout = config.health_timeout + self.paths = config.health_paths + self._enabled = config.health_enabled + + @property + def name(self) -> str: + return "health" + + @property + def enabled(self) -> bool: + return self._enabled + + def should_process(self, workload: Workload) -> bool: + """Only process workloads with addresses and ports.""" + return bool(workload.addresses) and bool(workload.ports) + + def process(self, workload: Workload) -> Optional[dict]: + """ + Probe health endpoints on the workload. + + Tries each address:port combination with each health path. + Returns on first successful response. + """ + if not self.should_process(workload): + return HealthResult( + healthy=False, + error="No addresses or ports available" + ).to_dict() + + # Build list of URLs to try + urls_to_try = [] + for addr in workload.addresses: + for port in workload.ports: + for path in self.paths: + urls_to_try.append(f"http://{addr}:{port}{path}") + + logger.info("[health] Probing (%s) URLs for workload %s", ','.join(urls_to_try), workload.name) + + # Try each URL + for url in urls_to_try: + result = self._probe_url(url) + if result.healthy: + logger.info( + "[health] %s is healthy at %s (%.0fms)", + workload.name, + result.endpoint, + result.response_time_ms or 0 + ) + return result.to_dict() + else: + logger.warning( + "[health] %s health probe failed at %s: %s", + workload.name, + result.endpoint, + result.error or f"HTTP {result.status_code}" + ) + + # All probes failed + logger.warning("[health] %s is unhealthy - no endpoints responded", workload.name) + return HealthResult( + healthy=False, + error=f"No health endpoints responded (tried {len(urls_to_try)} URLs)" + ).to_dict() + + def _probe_url(self, url: str) -> HealthResult: + """Probe a single URL.""" + try: + start = time.time() + response = requests.get(url, timeout=self.timeout, allow_redirects=True) + elapsed_ms = (time.time() - start) * 1000 + + # Consider 2xx and some 3xx as healthy + if response.status_code < 400: + return HealthResult( + healthy=True, + endpoint=url, + status_code=response.status_code, + response_time_ms=round(elapsed_ms, 2) + ) + else: + return HealthResult( + healthy=False, + endpoint=url, + status_code=response.status_code, + response_time_ms=round(elapsed_ms, 2), + error=f"HTTP {response.status_code}" + ) + + except requests.exceptions.Timeout: + return HealthResult( + healthy=False, + endpoint=url, + error="Timeout" + ) + except requests.exceptions.ConnectionError as e: + return HealthResult( + healthy=False, + endpoint=url, + error=f"Connection error: {str(e)[:100]}" + ) + except Exception as e: + return HealthResult( + healthy=False, + endpoint=url, + error=f"Error: {str(e)[:100]}" + ) diff --git a/discovery/inspector/processors/interface.py b/discovery/inspector/processors/interface.py new file mode 100644 index 000000000..76a2cdd55 --- /dev/null +++ b/discovery/inspector/processors/interface.py @@ -0,0 +1,55 @@ +""" +Processor interface. +""" + +from abc import ABC, abstractmethod +from typing import Optional + +from models import Workload + + +class ProcessorInterface(ABC): + """ + Abstract interface for metadata processors. + + Each processor extracts specific metadata from workloads. + """ + + @property + @abstractmethod + def name(self) -> str: + """Processor name (used as metadata key).""" + pass + + @property + def enabled(self) -> bool: + """Whether this processor is enabled.""" + return True + + @abstractmethod + def process(self, workload: Workload) -> Optional[dict]: + """ + Process a workload and extract metadata. + + Args: + workload: The workload to process + + Returns: + dict of metadata to store, or None if processing failed/skipped + """ + pass + + @abstractmethod + def should_process(self, workload: Workload) -> bool: + """ + Check if this processor should handle the workload. + + Override to add filtering logic (e.g., by labels, runtime type). + + Args: + workload: The workload to check + + Returns: + True if processor should handle this workload + """ + pass diff --git a/discovery/inspector/processors/openapi.py b/discovery/inspector/processors/openapi.py new file mode 100644 index 000000000..74e4a28ce --- /dev/null +++ b/discovery/inspector/processors/openapi.py @@ -0,0 +1,188 @@ +""" +OpenAPI specification processor. + +Discovers and extracts OpenAPI/Swagger specifications from workloads. +""" + +import json +from dataclasses import dataclass, field, asdict +from datetime import datetime +from typing import Optional + +import requests + +from config import ProcessorConfig, logger +from models import Workload +from processors.interface import ProcessorInterface + + +# ==================== Result Model ==================== + +@dataclass +class OpenAPIResult: + """OpenAPI discovery result.""" + found: bool + endpoint: Optional[str] = None + version: Optional[str] = None + title: Optional[str] = None + paths_count: Optional[int] = None + spec: Optional[dict] = None + error: Optional[str] = None + discovered_at: str = field(default_factory=lambda: datetime.utcnow().isoformat()) + + def to_dict(self) -> dict: + # Don't include full spec in serialization by default + d = asdict(self) + if d.get("spec"): + d["has_spec"] = True + del d["spec"] + return d + + def to_json(self) -> str: + return json.dumps(self.to_dict()) + + +# ==================== Processor ==================== + +class OpenAPIProcessor(ProcessorInterface): + """ + OpenAPI specification processor. + + Discovers OpenAPI/Swagger specs from workloads and extracts: + - Spec location + - API version and title + - Number of endpoints + - Optionally stores the full spec + """ + + def __init__(self, config: ProcessorConfig): + self.config = config + self.timeout = config.openapi_timeout + self.paths = config.openapi_paths + self._enabled = config.openapi_enabled + + @property + def name(self) -> str: + return "openapi" + + @property + def enabled(self) -> bool: + return self._enabled + + def should_process(self, workload: Workload) -> bool: + """Only process workloads with addresses and ports.""" + # Could also check for labels like 'openapi=true' or 'api=true' + return bool(workload.addresses) and bool(workload.ports) + + def process(self, workload: Workload) -> Optional[dict]: + """ + Discover OpenAPI specification from workload. + + Tries common OpenAPI/Swagger endpoints and parses the spec. + """ + if not self.should_process(workload): + return OpenAPIResult( + found=False, + error="No addresses or ports available" + ).to_dict() + + # Build list of URLs to try + urls_to_try = [] + for addr in workload.addresses: + for port in workload.ports: + for path in self.paths: + url = f"http://{addr}:{port}{path}" + urls_to_try.append(url) + + # Try each URL + for url in urls_to_try: + result = self._fetch_spec(url) + if result.found: + logger.info( + "[openapi] %s has spec at %s (%s v%s, %d paths)", + workload.name, + result.endpoint, + result.title or "Untitled", + result.version or "?", + result.paths_count or 0 + ) + return result.to_dict() + + # No spec found + logger.debug("[openapi] %s has no OpenAPI spec", workload.name) + return OpenAPIResult( + found=False, + error=f"No OpenAPI spec found (tried {len(urls_to_try)} URLs)" + ).to_dict() + + def _fetch_spec(self, url: str) -> OpenAPIResult: + """Fetch and parse OpenAPI spec from URL.""" + try: + response = requests.get( + url, + timeout=self.timeout, + headers={"Accept": "application/json"} + ) + + if response.status_code != 200: + return OpenAPIResult(found=False, error=f"HTTP {response.status_code}") + + # Try to parse as JSON + try: + spec = response.json() + except Exception: + return OpenAPIResult(found=False, error="Invalid JSON") + + # Validate it looks like an OpenAPI spec + if not self._is_openapi_spec(spec): + return OpenAPIResult(found=False, error="Not an OpenAPI spec") + + # Extract metadata + return OpenAPIResult( + found=True, + endpoint=url, + version=self._get_version(spec), + title=self._get_title(spec), + paths_count=self._count_paths(spec), + spec=spec # Store full spec + ) + + except requests.exceptions.Timeout: + return OpenAPIResult(found=False, error="Timeout") + except requests.exceptions.ConnectionError: + return OpenAPIResult(found=False, error="Connection error") + except Exception as e: + return OpenAPIResult(found=False, error=str(e)[:100]) + + def _is_openapi_spec(self, spec: dict) -> bool: + """Check if dict looks like an OpenAPI/Swagger spec.""" + # OpenAPI 3.x + if "openapi" in spec and "paths" in spec: + return True + # Swagger 2.x + if "swagger" in spec and "paths" in spec: + return True + # OpenAPI 3.1 can have webhooks instead of paths + if "openapi" in spec and "webhooks" in spec: + return True + return False + + def _get_version(self, spec: dict) -> Optional[str]: + """Extract API version from spec.""" + # OpenAPI 3.x + if "openapi" in spec: + return spec.get("openapi") + # Swagger 2.x + if "swagger" in spec: + return spec.get("swagger") + return None + + def _get_title(self, spec: dict) -> Optional[str]: + """Extract API title from spec.""" + info = spec.get("info", {}) + return info.get("title") + + def _count_paths(self, spec: dict) -> int: + """Count number of paths/endpoints in spec.""" + paths = spec.get("paths", {}) + return len(paths) diff --git a/discovery/inspector/requirements.txt b/discovery/inspector/requirements.txt new file mode 100644 index 000000000..0ac92d9ce --- /dev/null +++ b/discovery/inspector/requirements.txt @@ -0,0 +1,6 @@ +# Core dependencies +etcd3 +protobuf>=3.20,<4 + +# HTTP requests for health checks and API discovery +requests diff --git a/discovery/inspector/storage/__init__.py b/discovery/inspector/storage/__init__.py new file mode 100644 index 000000000..b2e5dd5e4 --- /dev/null +++ b/discovery/inspector/storage/__init__.py @@ -0,0 +1,23 @@ +""" +Storage module for inspector. +""" + +from storage.interface import StorageInterface +from storage.etcd import EtcdStorage +from config import logger, Config + + +def create_storage(config: Config) -> StorageInterface: + """ + Create storage backend from config. + + Currently only supports etcd. + """ + logger.info("Connecting to etcd at %s:%d", config.etcd.host, config.etcd.port) + + storage = EtcdStorage(config=config.etcd) + storage.connect() + + logger.info("Storage initialized") + + return storage diff --git a/discovery/inspector/storage/etcd.py b/discovery/inspector/storage/etcd.py new file mode 100644 index 000000000..251cb1424 --- /dev/null +++ b/discovery/inspector/storage/etcd.py @@ -0,0 +1,193 @@ +""" +etcd storage for inspector - watches workloads, writes metadata. + +Key structure: + /discovery/workloads/{id} → Workload JSON (watched by inspector) + /discovery/metadata/{id}/{processor} → Metadata JSON (written by inspector) + +Uses native etcd3 library for proper gRPC watch support. +""" + +import json +from typing import Optional, Generator, Tuple + +import etcd3 +from etcd3.events import PutEvent, DeleteEvent + +from config import EtcdConfig, logger +from models import Workload +from storage.interface import StorageInterface + + +class EtcdStorage(StorageInterface): + """ + etcd storage for watching workload entries and writing metadata. + + Uses native etcd3 watch for real-time updates. + """ + + def __init__(self, config: EtcdConfig): + self.host = config.host + self.port = config.port + self.workloads_prefix = config.workloads_prefix + self.metadata_prefix = config.metadata_prefix + self.username = config.username + self.password = config.password + self._client: Optional[etcd3.Etcd3Client] = None + + @property + def client(self) -> etcd3.Etcd3Client: + if self._client is None: + self._client = etcd3.client( + host=self.host, + port=self.port, + user=self.username, + password=self.password, + ) + return self._client + + def connect(self) -> bool: + """Connect to etcd.""" + try: + self.client.status() + logger.info("Connected to etcd at %s:%d", self.host, self.port) + return True + except Exception as e: + logger.error("Failed to connect to etcd: %s", e) + return False + + def close(self): + """Close connection.""" + if self._client: + self._client.close() + self._client = None + + # ==================== Workload Reading ==================== + + def list_workloads(self) -> list: + """List all workloads from storage.""" + workloads = [] + try: + for value, meta in self.client.get_prefix(self.workloads_prefix): + if not value: + continue + + key = meta.key.decode() if isinstance(meta.key, bytes) else meta.key + workload_id = key.replace(self.workloads_prefix, "") + + try: + data = json.loads(value.decode() if isinstance(value, bytes) else value) + workloads.append(Workload.from_dict(data)) + except (json.JSONDecodeError, KeyError) as e: + logger.warning("Failed to parse workload %s: %s", workload_id, e) + except Exception as e: + logger.error("Failed to list workloads: %s", e) + return workloads + + def get_workload(self, workload_id: str) -> Optional[Workload]: + """Get a specific workload by ID.""" + try: + key = f"{self.workloads_prefix}{workload_id}" + value, _ = self.client.get(key) + if value: + data = json.loads(value.decode() if isinstance(value, bytes) else value) + return Workload.from_dict(data) + except Exception as e: + logger.warning("Failed to get workload %s: %s", workload_id, e) + return None + + def watch_workloads(self) -> Generator[Tuple[str, Optional[Workload]], None, None]: + """ + Watch for workload changes using native etcd3 watch. + + Yields: + Tuple of (event_type, workload) where event_type is 'PUT' or 'DELETE' + """ + # First, yield all existing workloads + logger.info("Initial sync: loading existing workloads...") + for workload in self.list_workloads(): + yield ("PUT", workload) + + logger.info("Starting watch on %s", self.workloads_prefix) + + # Watch for changes + while True: + try: + events_iterator, cancel = self.client.watch_prefix(self.workloads_prefix) + + for event in events_iterator: + key = event.key.decode() if isinstance(event.key, bytes) else event.key + workload_id = key.replace(self.workloads_prefix, "") + + if isinstance(event, PutEvent): + try: + value = event.value.decode() if isinstance(event.value, bytes) else event.value + data = json.loads(value) + workload = Workload.from_dict(data) + logger.info("Watch: workload updated: %s", workload.name) + yield ("PUT", workload) + except (json.JSONDecodeError, KeyError) as e: + logger.warning("Watch: failed to parse workload %s: %s", workload_id, e) + + elif isinstance(event, DeleteEvent): + # Create a minimal workload object for delete event + logger.info("Watch: workload removed: %s", workload_id[:12]) + deleted_workload = Workload( + id=workload_id, + name=workload_id[:12], + hostname=workload_id[:12], + runtime="unknown", + workload_type="unknown", + addresses=[], + ports=[], + ) + yield ("DELETE", deleted_workload) + + cancel() + except Exception as e: + logger.warning("Watch error: %s, reconnecting...", e) + import time + time.sleep(1) + + # ==================== Metadata Writing ==================== + + def set_metadata(self, workload_id: str, processor_key: str, data: dict) -> bool: + """ + Write metadata for a workload. + + Args: + workload_id: The workload ID + processor_key: Processor name (e.g., 'health', 'openapi') + data: Metadata dict to store + + Stores at: /discovery/metadata/{workload_id}/{processor_key} + """ + try: + key = f"{self.metadata_prefix}{workload_id}/{processor_key}" + self.client.put(key, json.dumps(data)) + logger.debug("Set metadata %s for %s", processor_key, workload_id[:12]) + return True + except Exception as e: + logger.error("Failed to set metadata %s for %s: %s", processor_key, workload_id[:12], e) + return False + + def delete_metadata(self, workload_id: str, processor_key: Optional[str] = None) -> bool: + """ + Delete metadata for a workload. + + Args: + workload_id: The workload ID + processor_key: Optional specific processor to delete, or all metadata if None + """ + try: + if processor_key: + key = f"{self.metadata_prefix}{workload_id}/{processor_key}" + self.client.delete(key) + else: + prefix = f"{self.metadata_prefix}{workload_id}/" + self.client.delete_prefix(prefix) + logger.debug("Deleted metadata for %s", workload_id[:12]) + return True + except Exception as e: + logger.error("Failed to delete metadata for %s: %s", workload_id[:12], e) + return False diff --git a/discovery/inspector/storage/interface.py b/discovery/inspector/storage/interface.py new file mode 100644 index 000000000..348c31a22 --- /dev/null +++ b/discovery/inspector/storage/interface.py @@ -0,0 +1,92 @@ +""" +Storage interface for metadata inspector. + +Key structure: + /discovery/workloads/{id} → Workload JSON (watched by inspector) + /discovery/metadata/{id}/{processor} → Processor metadata JSON (written by inspector) + +The inspector watches workloads and writes metadata to a separate prefix. +""" + +from abc import ABC, abstractmethod +from typing import Optional, Generator, Tuple + +from models import Workload + + +class StorageInterface(ABC): + """ + Abstract interface for inspector storage. + + The inspector: + - Watches workloads from /discovery/workloads/{id} + - Writes metadata to /discovery/metadata/{id}/{processor} + """ + + @abstractmethod + def connect(self) -> bool: + """ + Connect to storage backend. + Returns True if connected successfully. + """ + pass + + @abstractmethod + def close(self): + """Close connection to storage backend.""" + pass + + # ==================== Workload Reading ==================== + + @abstractmethod + def list_workloads(self) -> list: + """List all workloads (ignores metadata keys).""" + pass + + @abstractmethod + def get_workload(self, workload_id: str) -> Optional[Workload]: + """Get a specific workload by ID.""" + pass + + @abstractmethod + def watch_workloads(self) -> Generator[Tuple[str, Optional[Workload]], None, None]: + """ + Watch for workload changes (ignores metadata changes). + + Yields: + Tuple of (event_type, workload) where event_type is 'PUT' or 'DELETE' + """ + pass + + # ==================== Metadata Writing ==================== + + @abstractmethod + def set_metadata(self, workload_id: str, processor_key: str, data: dict) -> bool: + """ + Write metadata for a workload. + + Args: + workload_id: The workload ID + processor_key: Processor name (e.g., 'health', 'openapi') + data: Metadata dict to store + + Writes to: /discovery/workloads/{workload_id}/metadata/{processor_key} + + Returns: + True on success + """ + pass + + @abstractmethod + def delete_metadata(self, workload_id: str, processor_key: Optional[str] = None) -> bool: + """ + Delete metadata for a workload. + + Args: + workload_id: The workload ID + processor_key: Optional specific processor to delete, or all metadata if None + + Returns: + True on success + """ + pass diff --git a/discovery/k8s.discovery.yaml b/discovery/k8s.discovery.yaml new file mode 100644 index 000000000..a2c2313ea --- /dev/null +++ b/discovery/k8s.discovery.yaml @@ -0,0 +1,419 @@ +# Discovery Service - Kubernetes Deployment +# Includes: etcd, watcher (with RBAC), server, test workloads, and network policy +# +# Usage: +# kubectl apply -f k8s.discovery.yaml +# kubectl wait --for=condition=ready pod -l app=etcd --timeout=60s +# kubectl wait --for=condition=ready pod -l app=discovery-watcher --timeout=60s +# kubectl wait --for=condition=ready pod -l app=discovery-server --timeout=60s +# kubectl wait --for=condition=ready pod -l discover=true -n team-a --timeout=60s +# kubectl wait --for=condition=ready pod -l discover=true -n team-b --timeout=60s +# +# Test: +# kubectl port-forward svc/discovery-server 8080:8080 & +# curl http://localhost:8080/workloads | jq . +# +# ============================================================================ +# ETCD +# ============================================================================ +--- +apiVersion: v1 +kind: Service +metadata: + name: discovery-etcd + namespace: default +spec: + ports: + - port: 2379 + targetPort: 2379 + selector: + app: discovery-etcd +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: discovery-etcd + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: discovery-etcd + template: + metadata: + labels: + app: discovery-etcd + spec: + containers: + - name: etcd + image: quay.io/coreos/etcd:v3.5.9 + command: + - etcd + - --listen-client-urls=http://0.0.0.0:2379 + - --advertise-client-urls=http://discovery-etcd:2379 + ports: + - containerPort: 2379 +# ============================================================================ +# WATCHER (with RBAC) +# ============================================================================ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: discovery-watcher + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: discovery-watcher +rules: + - apiGroups: [""] + resources: ["pods", "services", "namespaces"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: discovery-watcher +subjects: + - kind: ServiceAccount + name: discovery-watcher + namespace: default +roleRef: + kind: ClusterRole + name: discovery-watcher + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: discovery-watcher + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: discovery-watcher + template: + metadata: + labels: + app: discovery-watcher + spec: + serviceAccountName: discovery-watcher + containers: + - name: watcher + image: discovery-watcher:latest + imagePullPolicy: Never + env: + - name: RUNTIME + value: "kubernetes" + - name: KUBERNETES_IN_CLUSTER + value: "true" + - name: KUBERNETES_LABEL_KEY + value: "discover" + - name: KUBERNETES_LABEL_VALUE + value: "true" + - name: KUBERNETES_WATCH_SERVICES + value: "true" + - name: ETCD_HOST + value: "discovery-etcd" + - name: ETCD_PORT + value: "2379" + - name: ETCD_WORKLOADS_PREFIX + value: "/discovery/workloads/" +# ============================================================================ +# SERVER +# ============================================================================ +--- +apiVersion: v1 +kind: Service +metadata: + name: discovery-server + namespace: default +spec: + type: NodePort + ports: + - port: 8080 + targetPort: 8080 + nodePort: 30080 + selector: + app: discovery-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: discovery-server + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: discovery-server + template: + metadata: + labels: + app: discovery-server + spec: + containers: + - name: server + image: discovery-server:latest + imagePullPolicy: Never + ports: + - containerPort: 8080 + env: + - name: ETCD_HOST + value: "discovery-etcd" + - name: ETCD_PORT + value: "2379" + - name: ETCD_WORKLOADS_PREFIX + value: "/discovery/workloads/" + - name: ETCD_METADATA_PREFIX + value: "/discovery/metadata/" +# ============================================================================ +# INSPECTOR - Metadata extraction (health checks, OpenAPI discovery) +# ============================================================================ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: discovery-inspector + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: discovery-inspector + template: + metadata: + labels: + app: discovery-inspector + # Label used by NetworkPolicy to allow inspector ingress + role: inspector + spec: + containers: + - name: inspector + image: discovery-inspector:latest + imagePullPolicy: Never + env: + - name: ETCD_HOST + value: "discovery-etcd" + - name: ETCD_PORT + value: "2379" + - name: ETCD_WORKLOADS_PREFIX + value: "/discovery/workloads/" + - name: ETCD_METADATA_PREFIX + value: "/discovery/metadata/" + - name: HEALTH_ENABLED + value: "true" + - name: HEALTH_TIMEOUT + value: "5" + - name: OPENAPI_ENABLED + value: "true" + - name: OPENAPI_TIMEOUT + value: "10" + - name: PROCESSOR_WORKERS + value: "4" +# ============================================================================ +# TEST WORKLOADS (namespaces act as isolation groups) +# Pod DNS: {pod-ip-dashed}.{namespace}.pod (e.g., 10-244-0-5.team-a.pod) +# Service DNS: {service-name}.{namespace}.svc (e.g., svc-1.team-a.svc) +# ============================================================================ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: team-a +--- +apiVersion: v1 +kind: Namespace +metadata: + name: team-b +--- +# Service for service-1 pod +apiVersion: v1 +kind: Service +metadata: + name: svc-1 + namespace: team-a + labels: + discover: "true" +spec: + selector: + app: service-1 + ports: + - port: 80 + targetPort: 80 +--- +# Service for service-3 pod +apiVersion: v1 +kind: Service +metadata: + name: svc-3 + namespace: team-a + labels: + discover: "true" +spec: + selector: + app: service-3 + ports: + - port: 80 + targetPort: 80 +--- +# Service for service-2 pod +apiVersion: v1 +kind: Service +metadata: + name: svc-2 + namespace: team-b + labels: + discover: "true" +spec: + selector: + app: service-2 + ports: + - port: 80 + targetPort: 80 +--- +# Service for service-4 pod +apiVersion: v1 +kind: Service +metadata: + name: svc-4 + namespace: team-b + labels: + discover: "true" +spec: + selector: + app: service-4 + ports: + - port: 80 + targetPort: 80 +--- +apiVersion: v1 +kind: Pod +metadata: + name: service-1 + namespace: team-a + labels: + discover: "true" + app: service-1 +spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Pod +metadata: + name: service-3 + namespace: team-a + labels: + discover: "true" + app: service-3 +spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Pod +metadata: + name: service-2 + namespace: team-b + labels: + discover: "true" + app: service-2 +spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Pod +metadata: + name: service-4 + namespace: team-b + labels: + discover: "true" + app: service-4 +spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 +# ============================================================================ +# NETWORK POLICY - Allow inspector to reach discoverable pods +# ============================================================================ +--- +# Allow inspector (from default namespace) to reach team-a pods +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-inspector-ingress + namespace: team-a +spec: + podSelector: + matchLabels: + discover: "true" + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: default + podSelector: + matchLabels: + role: inspector + ports: + - protocol: TCP + port: 80 +--- +# Allow inspector (from default namespace) to reach team-b pods +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-inspector-ingress + namespace: team-b +spec: + podSelector: + matchLabels: + discover: "true" + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: default + podSelector: + matchLabels: + role: inspector + ports: + - protocol: TCP + port: 80 +# ============================================================================ +# NETWORK POLICY (optional - tests NetworkPolicy detection) +# ============================================================================ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: deny-all + namespace: team-a +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress diff --git a/discovery/server/Dockerfile b/discovery/server/Dockerfile new file mode 100644 index 000000000..50c586073 --- /dev/null +++ b/discovery/server/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY . . +RUN pip install --no-cache-dir -r requirements.txt + +EXPOSE 8080 + +CMD ["python", "main.py"] diff --git a/discovery/server/app.py b/discovery/server/app.py new file mode 100644 index 000000000..559fb0361 --- /dev/null +++ b/discovery/server/app.py @@ -0,0 +1,189 @@ +""" +HTTP API routes for service discovery. +Provides endpoints to query reachability across Docker, containerd, and Kubernetes. +""" + +import logging +from flask import Flask, jsonify, request + +from storage.interface import StorageInterface + +logger = logging.getLogger("discovery.server") + + +def create_app(storage: StorageInterface) -> Flask: + """Create and configure the Flask application.""" + app = Flask(__name__) + app.config['storage'] = storage + + @app.route('/discover', methods=['GET']) + @app.route('/reachable', methods=['GET']) + def discover(): + """ + Query reachability from a workload. + + Query parameters: + - from: Workload ID or hostname (required) + - runtime: Filter by runtime (docker, containerd, kubernetes) + - type: Filter by workload type (container, pod, service) + """ + from_identity = request.args.get('from') + if not from_identity: + return jsonify({"error": "Missing 'from' parameter (workload ID or hostname)"}), 400 + + runtime_filter = request.args.get('runtime') + type_filter = request.args.get('type') + + # Find source workload by ID, hostname, or name + source = storage.get(from_identity) + if not source: + source = storage.get_by_hostname(from_identity) + if not source: + source = storage.get_by_name(from_identity) + + if not source: + return jsonify({"error": f"Unknown workload: {from_identity}"}), 404 + + # Find reachable workloads + result = storage.find_reachable(source.id) + reachable = result.reachable + + # Apply filters + if runtime_filter: + reachable = [w for w in reachable if w.runtime == runtime_filter.lower()] + + if type_filter: + reachable = [w for w in reachable if w.workload_type == type_filter.lower()] + + return jsonify({ + "source": { + "id": source.id, + "name": source.name, + "hostname": source.hostname, + "runtime": source.runtime, + "type": source.workload_type, + "isolation_groups": list(source.isolation_groups), + "addresses": source.addresses, + "ports": source.ports, + "metadata": source.metadata or {}, + }, + "reachable": [ + { + "id": w.id, + "name": w.name, + "hostname": w.hostname, + "runtime": w.runtime, + "type": w.workload_type, + "isolation_groups": list(w.isolation_groups), + "addresses": w.addresses, + "ports": w.ports, + "labels": w.labels, + "metadata": w.metadata or {}, + } + for w in reachable + ], + "count": len(reachable), + "query": { + "from": from_identity, + "runtime_filter": runtime_filter, + "type_filter": type_filter, + } + }) + + @app.route('/workloads', methods=['GET']) + def list_workloads(): + """ + List all registered workloads. + + Query parameters: + - runtime: Filter by runtime + - group: Filter by isolation group + """ + runtime_filter = request.args.get('runtime') + group_filter = request.args.get('group') + + workloads = storage.list_all() + + if runtime_filter: + workloads = [w for w in workloads if w.runtime == runtime_filter] + + if group_filter: + workloads = [w for w in workloads if group_filter in w.isolation_groups] + + return jsonify({ + "workloads": [ + { + "id": w.id, + "name": w.name, + "hostname": w.hostname, + "runtime": w.runtime, + "type": w.workload_type, + "isolation_groups": list(w.isolation_groups), + "addresses": w.addresses, + "ports": w.ports, + "labels": w.labels, + "metadata": w.metadata or {}, + } + for w in workloads + ], + "count": len(workloads), + }) + + @app.route('/workload/', methods=['GET']) + def get_workload(workload_id: str): + """Get a specific workload by ID.""" + workload = storage.get(workload_id) + if not workload: + return jsonify({"error": f"Workload not found: {workload_id}"}), 404 + + return jsonify({ + "id": workload.id, + "name": workload.name, + "hostname": workload.hostname, + "runtime": workload.runtime, + "type": workload.workload_type, + "isolation_groups": list(workload.isolation_groups), + "addresses": workload.addresses, + "ports": workload.ports, + "labels": workload.labels, + "metadata": workload.metadata or {}, + }) + + @app.route('/health', methods=['GET']) + @app.route('/healthz', methods=['GET']) + def health(): + """Health check endpoint.""" + return jsonify({"status": "healthy"}) + + @app.route('/ready', methods=['GET']) + @app.route('/readyz', methods=['GET']) + def ready(): + """Readiness check endpoint.""" + try: + storage.list_all() + return jsonify({"status": "ready"}) + except Exception as e: + return jsonify({"error": f"Not ready: {e}"}), 503 + + @app.route('/stats', methods=['GET']) + def stats(): + """Get discovery statistics.""" + workloads = storage.list_all() + + by_runtime: dict = {} + by_type: dict = {} + all_groups: set = set() + + for w in workloads: + by_runtime[w.runtime] = by_runtime.get(w.runtime, 0) + 1 + by_type[w.workload_type] = by_type.get(w.workload_type, 0) + 1 + all_groups.update(w.isolation_groups) + + return jsonify({ + "total_workloads": len(workloads), + "by_runtime": by_runtime, + "by_type": by_type, + "isolation_groups": len(all_groups), + }) + + return app diff --git a/discovery/server/config.py b/discovery/server/config.py new file mode 100644 index 000000000..270558097 --- /dev/null +++ b/discovery/server/config.py @@ -0,0 +1,60 @@ +""" +Configuration for the discovery API server. +""" + +import logging +import os +import sys +from dataclasses import dataclass, field +from typing import Optional + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - [%(name)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + stream=sys.stdout, + force=True +) + +logger = logging.getLogger("discovery.server") + + +@dataclass +class EtcdConfig: + """etcd storage configuration.""" + host: str = field(default_factory=lambda: os.getenv("ETCD_HOST", "localhost")) + port: int = field(default_factory=lambda: int(os.getenv("ETCD_PORT", "2379"))) + workloads_prefix: str = field(default_factory=lambda: os.getenv("ETCD_WORKLOADS_PREFIX", "/discovery/workloads/")) + metadata_prefix: str = field(default_factory=lambda: os.getenv("ETCD_METADATA_PREFIX", "/discovery/metadata/")) + username: Optional[str] = field(default_factory=lambda: os.getenv("ETCD_USERNAME")) + password: Optional[str] = field(default_factory=lambda: os.getenv("ETCD_PASSWORD")) + + @property + def url(self) -> str: + return f"http://{self.host}:{self.port}" + + +@dataclass +class ServerConfig: + """HTTP server configuration.""" + host: str = field(default_factory=lambda: os.getenv("SERVER_HOST", "0.0.0.0")) + port: int = field(default_factory=lambda: int(os.getenv("SERVER_PORT", "8080"))) + debug: bool = field(default_factory=lambda: os.getenv("DEBUG", "false").lower() == "true") + + +@dataclass +class Config: + """Main configuration for the discovery server.""" + etcd: EtcdConfig = field(default_factory=EtcdConfig) + server: ServerConfig = field(default_factory=ServerConfig) + + @classmethod + def from_env(cls) -> "Config": + """Create configuration from environment variables.""" + return cls() + + +def load_config() -> Config: + """Load configuration from environment.""" + return Config.from_env() diff --git a/discovery/server/main.py b/discovery/server/main.py new file mode 100644 index 000000000..93cfa6e2d --- /dev/null +++ b/discovery/server/main.py @@ -0,0 +1,81 @@ +""" +Discovery API server entry point. +Serves HTTP API for querying workload reachability from etcd storage. +""" + +import signal +import sys +from typing import Optional + +from config import load_config, Config, logger +from storage import create_storage, StorageInterface +from app import create_app + + +class DiscoveryServer: + """HTTP server for service discovery API.""" + + def __init__(self, config: Config): + self.config = config + self.storage: Optional[StorageInterface] = None + self.app = None + + def start(self) -> None: + """Start the discovery API server.""" + # Initialize storage + self.storage = create_storage(self.config) + + # Create Flask app + self.app = create_app(self.storage) + + # Start HTTP server (blocking) + logger.info("HTTP server listening on %s:%d", self.config.server.host, self.config.server.port) + from waitress import serve + serve( + self.app, + host=self.config.server.host, + port=self.config.server.port, + _quiet=True + ) + + def stop(self) -> None: + """Stop the discovery server.""" + if self.storage: + self.storage.close() + logger.info("Discovery server stopped") + + +def main(): + """Main entry point.""" + config = load_config() + + # Log configuration + logger.info("=" * 60) + logger.info("Discovery API Server") + logger.info("=" * 60) + logger.info("Storage: etcd @ %s:%d", config.etcd.host, config.etcd.port) + logger.info("Workloads prefix: %s", config.etcd.workloads_prefix) + logger.info("Metadata prefix: %s", config.etcd.metadata_prefix) + logger.info("HTTP server: %s:%d", config.server.host, config.server.port) + logger.info("=" * 60) + + server = DiscoveryServer(config) + + # Handle signals + def signal_handler(sig, frame): + logger.info("Received signal %s, shutting down...", sig) + server.stop() + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Start + try: + server.start() + except KeyboardInterrupt: + server.stop() + + +if __name__ == "__main__": + main() diff --git a/discovery/server/models.py b/discovery/server/models.py new file mode 100644 index 000000000..326394f03 --- /dev/null +++ b/discovery/server/models.py @@ -0,0 +1,97 @@ +""" +Shared data models for service discovery. +""" + +from dataclasses import dataclass, field, asdict +from enum import Enum +from typing import Optional +import json + + +@dataclass +class Workload: + """ + Unified workload representation across all runtimes. + + This is the core data structure stored in etcd and returned by queries. + """ + + # Identity + id: str # Unique ID (container ID, pod UID, service UID) + name: str # Human-readable name + hostname: str # What $HOSTNAME returns inside workload + + # Runtime info + runtime: str # Runtime enum value as string + workload_type: str # WorkloadType enum value as string + + # Location + node: Optional[str] = None # Node/host where running + namespace: Optional[str] = None # K8s namespace (None for Docker/containerd) + + # Network + addresses: list = field(default_factory=list) # ["{name}.{network}", ...] + isolation_groups: list = field(default_factory=list) # Networks (Docker/containerd) or namespaces (K8s) + ports: list = field(default_factory=list) # Exposed ports ["80", "443", ...] + + # Discovery metadata + labels: dict = field(default_factory=dict) + annotations: dict = field(default_factory=dict) + + # Scraped metadata (populated async) + metadata: Optional[dict] = None + + # Internal tracking + registrar: Optional[str] = None # Which watcher instance registered this + + def to_dict(self) -> dict: + """Convert to dictionary for JSON serialization.""" + return {k: v for k, v in asdict(self).items() if v is not None} + + def to_json(self) -> str: + """Convert to JSON string.""" + return json.dumps(self.to_dict()) + + @classmethod + def from_dict(cls, data: dict) -> "Workload": + """Create Workload from dictionary.""" + return cls( + id=data.get("id", ""), + name=data.get("name", ""), + hostname=data.get("hostname", ""), + runtime=data.get("runtime", ""), + workload_type=data.get("workload_type", ""), + node=data.get("node"), + namespace=data.get("namespace"), + addresses=data.get("addresses", []), + isolation_groups=data.get("isolation_groups", []), + ports=data.get("ports", []), + labels=data.get("labels", {}), + annotations=data.get("annotations", {}), + metadata=data.get("metadata"), + registrar=data.get("registrar"), + ) + + @classmethod + def from_json(cls, json_str: str) -> "Workload": + """Create Workload from JSON string.""" + return cls.from_dict(json.loads(json_str)) + + +@dataclass +class ReachabilityResult: + """Result of a reachability query.""" + + caller: Workload + reachable: list # List[Workload] + count: int = 0 + + def __post_init__(self): + self.count = len(self.reachable) + + def to_dict(self) -> dict: + return { + "caller": self.caller.to_dict(), + "reachable": [w.to_dict() for w in self.reachable], + "count": self.count, + } diff --git a/discovery/server/requirements.txt b/discovery/server/requirements.txt new file mode 100644 index 000000000..3cb06c09a --- /dev/null +++ b/discovery/server/requirements.txt @@ -0,0 +1,5 @@ +# Core dependencies +etcd3 +protobuf>=3.20,<4 +flask +waitress diff --git a/discovery/server/storage/__init__.py b/discovery/server/storage/__init__.py new file mode 100644 index 000000000..f781b8aa1 --- /dev/null +++ b/discovery/server/storage/__init__.py @@ -0,0 +1,21 @@ +"""Storage module for the discovery API server.""" + +from storage.interface import StorageInterface +from storage.etcd import EtcdStorage +from config import logger, Config + + +def create_storage(config: Config) -> StorageInterface: + """ + Create storage backend from config. + + Currently only supports etcd. + """ + logger.info("Connecting to etcd at %s:%d", config.etcd.host, config.etcd.port) + + storage = EtcdStorage(config=config.etcd) + storage.connect() + + logger.info("Storage initialized") + + return storage diff --git a/discovery/server/storage/etcd.py b/discovery/server/storage/etcd.py new file mode 100644 index 000000000..324324757 --- /dev/null +++ b/discovery/server/storage/etcd.py @@ -0,0 +1,505 @@ +""" +etcd-based storage for discovery server (read-only). + +Key structure: + /discovery/workloads/{id} → Workload JSON + /discovery/metadata/{id}/{processor} → Processor metadata JSON + +Uses native etcd3 library for proper gRPC watch support. +Indices are built in-memory and updated via etcd watch. +""" + +import json +import threading +from collections import defaultdict +from typing import Optional + +import etcd3 +from etcd3.events import PutEvent, DeleteEvent + +from models import Workload, ReachabilityResult +from config import EtcdConfig, logger +from storage.interface import StorageInterface + + +class EtcdStorage(StorageInterface): + """ + etcd storage with in-memory indices for fast queries. + + Uses native etcd3 watch for real-time updates. + Watches two separate prefixes: + - workloads_prefix: /discovery/workloads/ (workload data from watcher) + - metadata_prefix: /discovery/metadata/ (metadata from inspector) + """ + + def __init__(self, config: EtcdConfig): + self.host = config.host + self.port = config.port + self.workloads_prefix = config.workloads_prefix + self.metadata_prefix = config.metadata_prefix + self.username = config.username + self.password = config.password + + self._client: Optional[etcd3.Etcd3Client] = None + self._connected = False + + # In-memory indices (protected by lock) + self._lock = threading.RLock() + self._workloads: dict[str, Workload] = {} # id → Workload + self._metadata: dict[str, dict] = {} # id → {processor: data} + self._by_hostname: dict[str, str] = {} # hostname → id + self._by_name: dict[str, str] = {} # "namespace/name" or "name" → id + self._by_group: dict[str, set[str]] = defaultdict(set) # group → {ids} + + # Watch threads + self._workload_watch_thread: Optional[threading.Thread] = None + self._metadata_watch_thread: Optional[threading.Thread] = None + self._stop_watch = threading.Event() + + @property + def client(self) -> etcd3.Etcd3Client: + if self._client is None: + self._client = etcd3.client( + host=self.host, + port=self.port, + user=self.username, + password=self.password, + ) + return self._client + + def connect(self) -> bool: + """Connect to etcd and start watching.""" + try: + self.client.status() + self._connected = True + logger.info("Connected to etcd at %s:%d", self.host, self.port) + + # Initial load + self._load_workloads() + self._load_metadata() + + # Start watch threads + self._start_watches() + + return True + except Exception as e: + logger.error("Failed to connect to etcd: %s", e) + return False + + def close(self): + """Stop watches and close connection.""" + self._stop_watch.set() + + if self._workload_watch_thread: + self._workload_watch_thread.join(timeout=5) + if self._metadata_watch_thread: + self._metadata_watch_thread.join(timeout=5) + + if self._client: + self._client.close() + self._client = None + + self._connected = False + logger.info("Storage closed") + + # ==================== Read Operations ==================== + + def get(self, workload_id: str) -> Optional[Workload]: + """Get workload by ID from index.""" + with self._lock: + workload = self._workloads.get(workload_id) + if workload: + return self._workload_with_metadata(workload) + return None + + def get_by_hostname(self, hostname: str) -> Optional[Workload]: + """Get workload by hostname.""" + with self._lock: + workload_id = self._by_hostname.get(hostname) + if workload_id: + workload = self._workloads.get(workload_id) + if workload: + return self._workload_with_metadata(workload) + return None + + def get_by_name(self, name: str, namespace: str = None) -> Optional[Workload]: + """Get workload by name (and namespace).""" + with self._lock: + key = f"{namespace}/{name}" if namespace else name + workload_id = self._by_name.get(key) + if not workload_id: + # Try without namespace + workload_id = self._by_name.get(name) + + if workload_id: + workload = self._workloads.get(workload_id) + if workload: + return self._workload_with_metadata(workload) + return None + + def list_all(self, runtime: str = None, label_filter: dict = None) -> list: + """List all workloads with optional filters.""" + with self._lock: + results = [] + for workload in self._workloads.values(): + # Filter by runtime + if runtime and workload.runtime != runtime: + continue + + # Filter by labels + if label_filter: + match = all( + workload.labels.get(k) == v + for k, v in label_filter.items() + ) + if not match: + continue + + results.append(self._workload_with_metadata(workload)) + + return results + + def _workload_with_metadata(self, workload: Workload) -> Workload: + """Return a copy of workload with merged metadata.""" + metadata = self._metadata.get(workload.id, {}) + if metadata: + # Merge stored metadata with any existing workload metadata + workload_meta = workload.metadata or {} + merged = {**workload_meta, **metadata} + return Workload( + id=workload.id, + name=workload.name, + hostname=workload.hostname, + runtime=workload.runtime, + workload_type=workload.workload_type, + node=workload.node, + namespace=workload.namespace, + addresses=workload.addresses, + isolation_groups=workload.isolation_groups, + ports=workload.ports, + labels=workload.labels, + annotations=workload.annotations, + metadata=merged, + registrar=workload.registrar, + ) + return workload + + # ==================== Reachability Queries ==================== + + def find_reachable(self, caller_identity: str) -> ReachabilityResult: + """ + Find all workloads reachable from caller. + + Reachability is based on shared isolation groups (networks/namespaces). + Addresses are filtered to only include those in shared isolation groups. + """ + # Find caller workload + caller = self._identify_workload(caller_identity) + if not caller: + raise ValueError(f"Caller not found: {caller_identity}") + + with self._lock: + # Get caller's effective isolation groups + caller_groups = set(caller.isolation_groups) + + if not caller_groups: + return ReachabilityResult(caller=caller, reachable=[]) + + # Find all workloads sharing at least one group + reachable_ids = set() + for group in caller_groups: + reachable_ids.update(self._by_group.get(group, set())) + + # Remove caller + reachable_ids.discard(caller.id) + + # Build result list with filtered addresses + reachable = [] + for wid in reachable_ids: + workload = self._workloads.get(wid) + if not workload: + continue + + # Find shared groups between caller and this workload + workload_groups = set(workload.isolation_groups) + shared_groups = caller_groups & workload_groups + + # Filter addresses to only include those in shared groups + filtered_addresses = self._filter_addresses(workload.addresses, shared_groups) + + # Create a copy with filtered addresses and metadata + metadata = self._metadata.get(wid, {}) + workload_meta = workload.metadata or {} + filtered_workload = Workload( + id=workload.id, + name=workload.name, + hostname=workload.hostname, + runtime=workload.runtime, + workload_type=workload.workload_type, + node=workload.node, + namespace=workload.namespace, + addresses=filtered_addresses, + isolation_groups=list(shared_groups), + ports=workload.ports, + labels=workload.labels, + annotations=workload.annotations, + metadata={**workload_meta, **metadata}, + registrar=workload.registrar, + ) + reachable.append(filtered_workload) + + # Sort by name + reachable.sort(key=lambda w: w.name) + + return ReachabilityResult(caller=caller, reachable=reachable) + + def _filter_addresses(self, addresses: list, shared_groups: set) -> list: + """Filter addresses to only those in shared isolation groups.""" + filtered = [] + for addr in addresses: + parts = addr.split(".") + if len(parts) >= 3 and parts[-1] in ("pod", "svc"): + # Kubernetes format: extract namespace + namespace = parts[-2] + if namespace in shared_groups: + filtered.append(addr) + elif len(parts) == 2: + # Docker format: {name}.{network} + network = parts[1] + if network in shared_groups: + filtered.append(addr) + else: + # Keep addresses that don't follow expected formats + filtered.append(addr) + return filtered + + # ==================== Internal Methods ==================== + + def _identify_workload(self, identity: str) -> Optional[Workload]: + """Find workload by hostname, name, or ID.""" + with self._lock: + # Try hostname first (most common for $HOSTNAME) + if identity in self._by_hostname: + wid = self._by_hostname[identity] + return self._workload_with_metadata(self._workloads[wid]) + + # Try name + if identity in self._by_name: + wid = self._by_name[identity] + return self._workload_with_metadata(self._workloads[wid]) + + # Try ID directly + if identity in self._workloads: + return self._workload_with_metadata(self._workloads[identity]) + + # Try ID prefix + for wid, workload in self._workloads.items(): + if wid.startswith(identity): + return self._workload_with_metadata(workload) + + return None + + def _load_workloads(self): + """Load all workloads from etcd.""" + logger.info("Loading workloads from %s", self.workloads_prefix) + + try: + for value, meta in self.client.get_prefix(self.workloads_prefix): + if not value: + continue + + key = meta.key.decode() if isinstance(meta.key, bytes) else meta.key + workload_id = key.replace(self.workloads_prefix, "") + + try: + data = json.loads(value.decode() if isinstance(value, bytes) else value) + workload = Workload.from_dict(data) + self._update_workload_index(workload_id, workload) + except (json.JSONDecodeError, KeyError) as e: + logger.warning("Failed to parse workload %s: %s", workload_id, e) + + logger.info("Loaded %d workloads", len(self._workloads)) + except Exception as e: + logger.error("Failed to load workloads: %s", e) + + def _load_metadata(self): + """Load all metadata from etcd.""" + logger.info("Loading metadata from %s", self.metadata_prefix) + + try: + for value, meta in self.client.get_prefix(self.metadata_prefix): + if not value: + continue + + key = meta.key.decode() if isinstance(meta.key, bytes) else meta.key + relative_key = key.replace(self.metadata_prefix, "") + parts = relative_key.split("/") + + if len(parts) >= 2: + workload_id = parts[0] + processor = parts[1] + + try: + data = json.loads(value.decode() if isinstance(value, bytes) else value) + with self._lock: + if workload_id not in self._metadata: + self._metadata[workload_id] = {} + self._metadata[workload_id][processor] = data + except json.JSONDecodeError as e: + logger.warning("Failed to parse metadata %s/%s: %s", workload_id, processor, e) + + logger.info("Loaded metadata for %d workloads", len(self._metadata)) + except Exception as e: + logger.error("Failed to load metadata: %s", e) + + def _start_watches(self): + """Start watch threads for workloads and metadata.""" + self._stop_watch.clear() + + self._workload_watch_thread = threading.Thread( + target=self._watch_workloads, + daemon=True, + name="workload-watcher" + ) + self._workload_watch_thread.start() + + self._metadata_watch_thread = threading.Thread( + target=self._watch_metadata, + daemon=True, + name="metadata-watcher" + ) + self._metadata_watch_thread.start() + + logger.info("Started watch threads") + + def _watch_workloads(self): + """Watch for workload changes.""" + while not self._stop_watch.is_set(): + try: + events_iterator, cancel = self.client.watch_prefix(self.workloads_prefix) + logger.info("Watching workloads at %s", self.workloads_prefix) + + for event in events_iterator: + if self._stop_watch.is_set(): + break + + key = event.key.decode() if isinstance(event.key, bytes) else event.key + workload_id = key.replace(self.workloads_prefix, "") + + if isinstance(event, PutEvent): + try: + value = event.value.decode() if isinstance(event.value, bytes) else event.value + data = json.loads(value) + workload = Workload.from_dict(data) + self._update_workload_index(workload_id, workload) + logger.info("Watch: updated workload %s", workload.name) + except (json.JSONDecodeError, KeyError) as e: + logger.warning("Watch: failed to parse workload %s: %s", workload_id, e) + + elif isinstance(event, DeleteEvent): + self._remove_workload_index(workload_id) + # Also remove associated metadata + with self._lock: + self._metadata.pop(workload_id, None) + logger.info("Watch: removed workload %s", workload_id[:12]) + + cancel() + except Exception as e: + if not self._stop_watch.is_set(): + logger.warning("Workload watch error: %s, reconnecting...", e) + import time + time.sleep(1) + + def _watch_metadata(self): + """Watch for metadata changes.""" + while not self._stop_watch.is_set(): + try: + events_iterator, cancel = self.client.watch_prefix(self.metadata_prefix) + logger.info("Watching metadata at %s", self.metadata_prefix) + + for event in events_iterator: + if self._stop_watch.is_set(): + break + + key = event.key.decode() if isinstance(event.key, bytes) else event.key + relative_key = key.replace(self.metadata_prefix, "") + parts = relative_key.split("/") + + if len(parts) < 2: + continue + + workload_id = parts[0] + processor = parts[1] + + if isinstance(event, PutEvent): + try: + value = event.value.decode() if isinstance(event.value, bytes) else event.value + data = json.loads(value) + with self._lock: + if workload_id not in self._metadata: + self._metadata[workload_id] = {} + self._metadata[workload_id][processor] = data + logger.debug("Watch: updated metadata %s for %s", processor, workload_id[:12]) + except json.JSONDecodeError as e: + logger.warning("Watch: failed to parse metadata: %s", e) + + elif isinstance(event, DeleteEvent): + with self._lock: + if workload_id in self._metadata: + self._metadata[workload_id].pop(processor, None) + logger.debug("Watch: removed metadata %s for %s", processor, workload_id[:12]) + + cancel() + except Exception as e: + if not self._stop_watch.is_set(): + logger.warning("Metadata watch error: %s, reconnecting...", e) + import time + time.sleep(1) + + def _update_workload_index(self, workload_id: str, workload: Workload): + """Update in-memory indices for a workload.""" + with self._lock: + # Remove old entries if exists + self._remove_workload_index(workload_id) + + # Add new entries + self._workloads[workload_id] = workload + + if workload.hostname: + self._by_hostname[workload.hostname] = workload_id + + # Index by name (with namespace if present) + if workload.namespace: + self._by_name[f"{workload.namespace}/{workload.name}"] = workload_id + self._by_name[workload.name] = workload_id + + # Index by isolation groups + for group in workload.isolation_groups: + self._by_group[group].add(workload_id) + + def _remove_workload_index(self, workload_id: str): + """Remove workload from all indices.""" + with self._lock: + workload = self._workloads.get(workload_id) + if not workload: + return + + # Remove from hostname index + if workload.hostname and self._by_hostname.get(workload.hostname) == workload_id: + del self._by_hostname[workload.hostname] + + # Remove from name index + name_key = f"{workload.namespace}/{workload.name}" if workload.namespace else workload.name + if self._by_name.get(name_key) == workload_id: + del self._by_name[name_key] + if self._by_name.get(workload.name) == workload_id: + del self._by_name[workload.name] + + # Remove from group indices + for group in workload.isolation_groups: + self._by_group[group].discard(workload_id) + if not self._by_group[group]: + del self._by_group[group] + + # Remove from main store + del self._workloads[workload_id] diff --git a/discovery/server/storage/interface.py b/discovery/server/storage/interface.py new file mode 100644 index 000000000..b677b8cca --- /dev/null +++ b/discovery/server/storage/interface.py @@ -0,0 +1,69 @@ +""" +Storage interface for discovery server (read-only). + +The server only needs to query workloads from etcd. +""" + +from abc import ABC, abstractmethod +from typing import Optional +from models import Workload, ReachabilityResult + + +class StorageInterface(ABC): + """ + Abstract interface for workload storage (read operations only). + + The server queries workloads and computes reachability. + """ + + @abstractmethod + def connect(self) -> bool: + """ + Connect to storage backend. + Returns True if connected successfully. + """ + pass + + @abstractmethod + def close(self): + """Close connection to storage backend.""" + pass + + @abstractmethod + def get(self, workload_id: str) -> Optional[Workload]: + """Get a workload by ID.""" + pass + + @abstractmethod + def get_by_hostname(self, hostname: str) -> Optional[Workload]: + """Get a workload by hostname.""" + pass + + @abstractmethod + def get_by_name(self, name: str, namespace: str = None) -> Optional[Workload]: + """Get a workload by name (and namespace for K8s).""" + pass + + @abstractmethod + def list_all(self, runtime: str = None, label_filter: dict = None) -> list: + """ + List all workloads, optionally filtered by runtime or labels. + Returns List[Workload]. + """ + pass + + @abstractmethod + def find_reachable( + self, + caller_identity: str, + ) -> ReachabilityResult: + """ + Find all workloads reachable from caller. + + Args: + caller_identity: Hostname, name, or ID of caller + + Returns: + ReachabilityResult with caller and reachable workloads + """ + pass diff --git a/discovery/watcher/Dockerfile b/discovery/watcher/Dockerfile new file mode 100644 index 000000000..cd5034f00 --- /dev/null +++ b/discovery/watcher/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.11-slim + +# Ensure Python output is sent straight to terminal without buffering +ENV PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY . . +RUN pip install --no-cache-dir -r requirements.txt + +CMD ["python", "main.py"] diff --git a/discovery/watcher/config.py b/discovery/watcher/config.py new file mode 100644 index 000000000..b7d69d613 --- /dev/null +++ b/discovery/watcher/config.py @@ -0,0 +1,84 @@ +""" +Configuration for multi-runtime service discovery. +Supports Docker, containerd, and Kubernetes runtimes with etcd storage. +""" + +import logging +import os +import sys +from dataclasses import dataclass, field +from typing import Optional + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - [%(name)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + stream=sys.stdout, + force=True +) + +logger = logging.getLogger("discovery") + + +@dataclass +class EtcdConfig: + """etcd storage configuration.""" + host: str = field(default_factory=lambda: os.getenv("ETCD_HOST", "localhost")) + port: int = field(default_factory=lambda: int(os.getenv("ETCD_PORT", "2379"))) + workloads_prefix: str = field(default_factory=lambda: os.getenv("ETCD_WORKLOADS_PREFIX", "/discovery/workloads/")) + username: Optional[str] = field(default_factory=lambda: os.getenv("ETCD_USERNAME")) + password: Optional[str] = field(default_factory=lambda: os.getenv("ETCD_PASSWORD")) + + @property + def url(self) -> str: + return f"http://{self.host}:{self.port}" + + +@dataclass +class DockerConfig: + """Docker runtime configuration.""" + socket: str = field(default_factory=lambda: os.getenv("DOCKER_SOCKET", "unix:///var/run/docker.sock")) + label_key: str = field(default_factory=lambda: os.getenv("DOCKER_LABEL_KEY", "discover")) + label_value: str = field(default_factory=lambda: os.getenv("DOCKER_LABEL_VALUE", "true")) + + +@dataclass +class ContainerdConfig: + """containerd runtime configuration.""" + socket: str = field(default_factory=lambda: os.getenv("CONTAINERD_SOCKET", "/run/containerd/containerd.sock")) + namespace: str = field(default_factory=lambda: os.getenv("CONTAINERD_NAMESPACE", "default")) + cni_state_dir: str = field(default_factory=lambda: os.getenv("CONTAINERD_CNI_STATE_DIR", "/var/lib/cni/results")) + label_key: str = field(default_factory=lambda: os.getenv("CONTAINERD_LABEL_KEY", "discover")) + label_value: str = field(default_factory=lambda: os.getenv("CONTAINERD_LABEL_VALUE", "true")) + + +@dataclass +class KubernetesConfig: + """Kubernetes runtime configuration.""" + kubeconfig: Optional[str] = field(default_factory=lambda: os.getenv("KUBECONFIG")) + namespace: Optional[str] = field(default_factory=lambda: os.getenv("KUBERNETES_NAMESPACE")) # None = all namespaces + in_cluster: bool = field(default_factory=lambda: os.getenv("KUBERNETES_IN_CLUSTER", "false").lower() == "true") + label_key: str = field(default_factory=lambda: os.getenv("KUBERNETES_LABEL_KEY", "discover")) + label_value: str = field(default_factory=lambda: os.getenv("KUBERNETES_LABEL_VALUE", "true")) + watch_services: bool = field(default_factory=lambda: os.getenv("KUBERNETES_WATCH_SERVICES", "true").lower() == "true") + + +@dataclass +class Config: + """Main configuration container for workload watcher.""" + runtime: str = field(default_factory=lambda: os.getenv("RUNTIME", "docker")) + etcd: EtcdConfig = field(default_factory=EtcdConfig) + docker: DockerConfig = field(default_factory=DockerConfig) + containerd: ContainerdConfig = field(default_factory=ContainerdConfig) + kubernetes: KubernetesConfig = field(default_factory=KubernetesConfig) + + @classmethod + def from_env(cls) -> "Config": + """Create configuration from environment variables.""" + return cls() + + +def load_config() -> Config: + """Load configuration from environment.""" + return Config.from_env() diff --git a/discovery/watcher/main.py b/discovery/watcher/main.py new file mode 100644 index 000000000..73bade98e --- /dev/null +++ b/discovery/watcher/main.py @@ -0,0 +1,178 @@ +""" +Workload watcher entry point. +Watches Docker, containerd, or Kubernetes for workload changes and syncs to etcd storage. +""" + +import signal +import sys +import threading +import time +from typing import Optional + +from config import load_config, Config, logger +from models import Workload, EventType +from storage import create_storage, StorageInterface +from runtimes import create_runtime, RuntimeAdapter + + +class WorkloadWatcher: + """ + Workload watcher coordinator. + + Watches a runtime for workload events and syncs to etcd storage. + """ + + def __init__(self, config: Config): + self.config = config + self.storage: Optional[StorageInterface] = None + self.runtime: Optional[RuntimeAdapter] = None + self._running = False + self._stop_event = threading.Event() + + def _handle_event(self, event_type: EventType, workload: Workload) -> None: + """Handle workload event from runtime.""" + if not self.storage: + return + + if event_type in (EventType.ADDED, EventType.MODIFIED, EventType.NETWORK_CHANGED): + self.storage.register(workload) + logger.info( + "[%s] %s workload: %s (%s) groups=%s", + event_type.value.upper(), + workload.runtime, + workload.name, + workload.id[:12], + list(workload.isolation_groups) + ) + elif event_type in (EventType.DELETED, EventType.PAUSED): + self.storage.deregister(workload.id) + logger.info( + "[%s] %s workload: %s (%s)", + event_type.value.upper(), + workload.runtime, + workload.name, + workload.id[:12] + ) + + def _sync_initial_state(self) -> None: + """Sync initial workload state from runtime and clean up stale entries.""" + logger.info("Syncing initial workload state...") + + try: + # Get current workloads from runtime + current_workloads = self.runtime.list_workloads() + current_ids = {w.id for w in current_workloads} + + # Get existing workloads from etcd + stored_ids = self.storage.list_workload_ids() + + # Find stale workloads (in etcd but not in runtime) + stale_ids = stored_ids - current_ids + if stale_ids: + logger.info("Cleaning up %d stale workloads...", len(stale_ids)) + for stale_id in stale_ids: + self.storage.deregister(stale_id) + logger.info("[CLEANUP] Removed stale workload: %s", stale_id[:12]) + + # Register current workloads + for workload in current_workloads: + self.storage.register(workload) + + logger.info( + "Synced %d workloads from %s (removed %d stale)", + len(current_workloads), + self.runtime.runtime_type.value, + len(stale_ids) + ) + except Exception as e: + logger.error("Failed to sync from %s: %s", self.runtime.runtime_type.value, e) + + def _start_watcher(self) -> None: + """Start event watcher for runtime in background thread.""" + try: + thread = threading.Thread( + target=self.runtime.watch_events, + args=(self._handle_event,), + daemon=True, + name=f"watcher-{self.runtime.runtime_type.value}" + ) + thread.start() + logger.info("Started event watcher for %s", self.runtime.runtime_type.value) + except Exception as e: + logger.error("Failed to start watcher for %s: %s", self.runtime.runtime_type.value, e) + + def start(self) -> None: + """Start the workload watcher.""" + self._running = True + + # Initialize storage + self.storage = create_storage(self.config) + + # Initialize runtime + self.runtime = create_runtime(self.config) + if not self.runtime: + logger.error("Failed to initialize runtime!") + sys.exit(1) + + # Sync and start watcher + self._sync_initial_state() + self._start_watcher() + + logger.info("Watcher running, press Ctrl+C to stop...") + + # Keep main thread alive + while not self._stop_event.is_set(): + self._stop_event.wait(timeout=1) + + def stop(self) -> None: + """Stop the workload watcher.""" + self._running = False + self._stop_event.set() + + # Stop watcher + if self.runtime: + try: + self.runtime.close() + except Exception as e: + logger.warning("Error stopping %s watcher: %s", self.runtime.runtime_type.value, e) + + # Stop storage + if self.storage: + self.storage.close() + + logger.info("Workload watcher stopped") + + +def main(): + """Main entry point.""" + config = load_config() + + # Log configuration + logger.info("=" * 60) + logger.info("Workload Watcher") + logger.info("=" * 60) + logger.info("Runtime: %s", config.runtime) + logger.info("Storage: etcd @ %s:%d", config.etcd.host, config.etcd.port) + logger.info("Workloads prefix: %s", config.etcd.workloads_prefix) + logger.info("=" * 60) + + watcher = WorkloadWatcher(config) + + # Handle signals + def signal_handler(sig, frame): + logger.info("Received signal %s, shutting down...", sig) + watcher.stop() + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Start + try: + watcher.start() + except KeyboardInterrupt: + watcher.stop() + + +if __name__ == "__main__": + main() diff --git a/discovery/watcher/models.py b/discovery/watcher/models.py new file mode 100644 index 000000000..2cc4369cd --- /dev/null +++ b/discovery/watcher/models.py @@ -0,0 +1,118 @@ +""" +Unified data models for multi-runtime service discovery. +""" + +from dataclasses import dataclass, field, asdict +from enum import Enum +from typing import Optional +import json + + +class Runtime(Enum): + DOCKER = "docker" + CONTAINERD = "containerd" + KUBERNETES = "kubernetes" + + +class WorkloadType(Enum): + CONTAINER = "container" # Docker/containerd container + POD = "pod" # K8s Pod + SERVICE = "service" # K8s Service (virtual endpoint) + + +class EventType(Enum): + """Types of workload events emitted by runtime adapters.""" + ADDED = "added" + MODIFIED = "modified" + DELETED = "deleted" + PAUSED = "paused" + NETWORK_CHANGED = "network_changed" + + +@dataclass +class Workload: + """ + Unified workload representation across all runtimes. + + This is the core data structure stored in etcd and returned by queries. + """ + + # Identity + id: str # Unique ID (container ID, pod UID, service UID) + name: str # Human-readable name + hostname: str # What $HOSTNAME returns inside workload + + # Runtime info + runtime: str # Runtime enum value as string + workload_type: str # WorkloadType enum value as string + + # Location + node: Optional[str] = None # Node/host where running + namespace: Optional[str] = None # K8s namespace (None for Docker/containerd) + + # Network + addresses: list = field(default_factory=list) # ["{name}.{network}", ...] + isolation_groups: list = field(default_factory=list) # Networks (Docker/containerd) or namespaces (K8s) + ports: list = field(default_factory=list) # Exposed ports ["80", "443", ...] + + # Discovery metadata + labels: dict = field(default_factory=dict) + annotations: dict = field(default_factory=dict) + + # Scraped metadata (populated async) + metadata: Optional[dict] = None + + # Internal tracking + registrar: Optional[str] = None # Which watcher instance registered this + + def to_dict(self) -> dict: + """Convert to dictionary for JSON serialization.""" + return {k: v for k, v in asdict(self).items() if v is not None} + + def to_json(self) -> str: + """Convert to JSON string.""" + return json.dumps(self.to_dict()) + + @classmethod + def from_dict(cls, data: dict) -> "Workload": + """Create Workload from dictionary.""" + return cls( + id=data.get("id", ""), + name=data.get("name", ""), + hostname=data.get("hostname", ""), + runtime=data.get("runtime", Runtime.DOCKER.value), + workload_type=data.get("workload_type", WorkloadType.CONTAINER.value), + node=data.get("node"), + namespace=data.get("namespace"), + addresses=data.get("addresses", []), + isolation_groups=data.get("isolation_groups", []), + ports=data.get("ports", []), + labels=data.get("labels", {}), + annotations=data.get("annotations", {}), + metadata=data.get("metadata"), + registrar=data.get("registrar"), + ) + + @classmethod + def from_json(cls, json_str: str) -> "Workload": + """Create Workload from JSON string.""" + return cls.from_dict(json.loads(json_str)) + + +@dataclass +class ReachabilityResult: + """Result of a reachability query.""" + + caller: Workload + reachable: list # List[Workload] + count: int = 0 + + def __post_init__(self): + self.count = len(self.reachable) + + def to_dict(self) -> dict: + return { + "caller": self.caller.to_dict(), + "reachable": [w.to_dict() for w in self.reachable], + "count": self.count, + } diff --git a/discovery/watcher/requirements.txt b/discovery/watcher/requirements.txt new file mode 100644 index 000000000..71009d8fd --- /dev/null +++ b/discovery/watcher/requirements.txt @@ -0,0 +1,13 @@ +# Core dependencies +etcd3 + +# Docker support +docker + +# containerd support (requires older protobuf for compatibility) +containerd +watchdog +protobuf>=3.20,<4 + +# Kubernetes support +kubernetes diff --git a/discovery/watcher/runtimes/__init__.py b/discovery/watcher/runtimes/__init__.py new file mode 100644 index 000000000..eff24b980 --- /dev/null +++ b/discovery/watcher/runtimes/__init__.py @@ -0,0 +1,36 @@ +"""Runtime adapters module for service discovery.""" + +import logging + +from config import Config +from runtimes.interface import RuntimeAdapter + +logger = logging.getLogger("discovery") + + +def create_runtime(config: Config) -> RuntimeAdapter: + """ + Create the configured runtime adapter. + + Returns connected adapter or None if connection fails. + """ + runtime_type = config.runtime + + # Initialize appropriate runtime adapter + if runtime_type == "docker": + from runtimes.docker import DockerAdapter + adapter = DockerAdapter(config.docker) + elif runtime_type == "containerd": + from runtimes.containerd import ContainerdAdapter + adapter = ContainerdAdapter(config.containerd) + elif runtime_type == "kubernetes": + from runtimes.kubernetes import KubernetesAdapter + adapter = KubernetesAdapter(config.kubernetes) + else: + logger.error("Unknown runtime: %s (use: docker, containerd, kubernetes)", runtime_type) + return None + + # Connect to runtime + adapter.connect() + logger.info("%s runtime connected", runtime_type) + return adapter diff --git a/discovery/watcher/runtimes/containerd.py b/discovery/watcher/runtimes/containerd.py new file mode 100644 index 000000000..8a4bb7993 --- /dev/null +++ b/discovery/watcher/runtimes/containerd.py @@ -0,0 +1,391 @@ +""" +containerd runtime adapter. + +Watches containerd for task events and reads CNI state for network info. +Uses the pycontainerd gRPC bindings (not a high-level client). +""" + +import json +import os +import threading +from pathlib import Path +from typing import Optional, Callable + +import grpc +from containerd.services.containers.v1 import containers_pb2_grpc, containers_pb2 +from containerd.services.tasks.v1 import tasks_pb2_grpc, tasks_pb2 +from containerd.services.events.v1 import events_pb2_grpc, events_pb2, unwrap +from containerd.types.task import task_pb2 +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +from models import Workload, Runtime, WorkloadType, EventType +from config import ContainerdConfig +from runtimes.interface import RuntimeAdapter + + +class ContainerdAdapter(RuntimeAdapter): + """ + containerd runtime adapter. + + Uses containerd gRPC API for container events. + Reads CNI state files for network information. + Optionally watches CNI state directory for network changes. + """ + + SOCKET_PATH = "/run/containerd/containerd.sock" + CNI_STATE_PATH = "/var/lib/cni/results" + + def __init__(self, config: ContainerdConfig): + """ + Initialize containerd adapter. + + Args: + config: ContainerdConfig instance + """ + self.socket_path = config.socket or self.SOCKET_PATH + self.namespace = config.namespace + self.cni_state_path = Path(config.cni_state_dir or self.CNI_STATE_PATH) + self.label_key = config.label_key + self.label_value = config.label_value + + self._channel = None + self._stop_event = threading.Event() + self._cni_observer = None + self._event_callback = None + + def _get_metadata(self): + """Get gRPC metadata with namespace.""" + return (('containerd-namespace', self.namespace),) + + @property + def runtime_type(self) -> Runtime: + return Runtime.CONTAINERD + + def connect(self) -> bool: + """Verify containerd connection.""" + try: + channel = grpc.insecure_channel(f'unix://{self.socket_path}') + containers_stub = containers_pb2_grpc.ContainersStub(channel) + # Test connection by listing containers + containers_stub.List( + containers_pb2.ListContainersRequest(), + metadata=self._get_metadata() + ) + channel.close() + return True + except Exception as e: + print(f"[containerd] Failed to connect: {e}") + return False + + def close(self): + """Close containerd connection and stop watchers.""" + self._stop_event.set() + if self._channel: + self._channel.close() + self._channel = None + if self._cni_observer: + self._cni_observer.stop() + self._cni_observer = None + + def list_workloads(self) -> list: + """List all discoverable containers.""" + workloads = [] + + try: + channel = grpc.insecure_channel(f'unix://{self.socket_path}') + containers_stub = containers_pb2_grpc.ContainersStub(channel) + tasks_stub = tasks_pb2_grpc.TasksStub(channel) + + # List all containers + response = containers_stub.List( + containers_pb2.ListContainersRequest(), + metadata=self._get_metadata() + ) + + for container in response.containers: + # Convert protobuf labels to dict + labels = dict(container.labels) if container.labels else {} + + # Check discover label + if labels.get(self.label_key) != self.label_value: + continue + + # Check if running by getting task status + try: + task_response = tasks_stub.Get( + tasks_pb2.GetRequest(container_id=container.id), + metadata=self._get_metadata() + ) + # Status enum: UNKNOWN=0, CREATED=1, RUNNING=2, STOPPED=3, PAUSED=4, PAUSING=5 + if task_response.process.status != task_pb2.RUNNING: + continue + except grpc.RpcError: + # No task means not running + continue + + workload = self._container_to_workload(container, labels) + if workload: + workloads.append(workload) + + channel.close() + + except Exception as e: + print(f"[containerd] Failed to list containers: {e}") + + return workloads + + def watch_events(self, callback: Callable[[str, Workload], None]) -> None: + """ + Watch containerd events and CNI changes. + + containerd events cover: task start/exit, container create/delete + CNI watcher covers: network connect/disconnect (via file changes) + """ + self._stop_event.clear() + self._event_callback = callback + + # Start CNI file watcher in background + if self.cni_state_path.exists(): + self._start_cni_watcher(callback) + + # Watch containerd events via gRPC + try: + channel = grpc.insecure_channel(f'unix://{self.socket_path}') + events_stub = events_pb2_grpc.EventsStub(channel) + + # Subscribe to all events (filter by namespace via metadata) + for envelope in events_stub.Subscribe( + events_pb2.SubscribeRequest(), + metadata=self._get_metadata() + ): + if self._stop_event.is_set(): + break + + # Unwrap the event to get the actual event object + try: + event = unwrap(envelope) + topic = envelope.topic + + # Extract container ID from the event + container_id = getattr(event, 'container_id', None) or getattr(event, 'id', None) + + if not container_id: + continue + + if topic == "/tasks/start": + workload = self._get_workload(container_id) + if workload: + callback(EventType.ADDED, workload) + + elif topic in ("/tasks/exit", "/tasks/delete", "/containers/delete"): + # Create minimal workload for deletion + workload = Workload( + id=container_id, + name=container_id[:12], + hostname=container_id[:12], + runtime=Runtime.CONTAINERD.value, + workload_type=WorkloadType.CONTAINER.value, + ) + callback(EventType.DELETED, workload) + + elif topic == "/tasks/paused": + # Paused containers are not reachable - remove from discovery + workload = Workload( + id=container_id, + name=container_id[:12], + hostname=container_id[:12], + runtime=Runtime.CONTAINERD.value, + workload_type=WorkloadType.CONTAINER.value, + ) + callback(EventType.PAUSED, workload) + + elif topic == "/tasks/resumed": + # Resumed container is reachable again - re-add to discovery + workload = self._get_workload(container_id) + if workload: + callback(EventType.ADDED, workload) + + except Exception as e: + print(f"[containerd] Failed to process event: {e}") + continue + + channel.close() + + except Exception as e: + if not self._stop_event.is_set(): + print(f"[containerd] Event watch error: {e}") + + def _get_workload(self, identity: str) -> Optional[Workload]: + """Get container by hostname, name, or ID.""" + try: + channel = grpc.insecure_channel(f'unix://{self.socket_path}') + containers_stub = containers_pb2_grpc.ContainersStub(channel) + + # List all and filter (containerd doesn't have get-by-name) + response = containers_stub.List( + containers_pb2.ListContainersRequest(), + metadata=self._get_metadata() + ) + + for container in response.containers: + labels = dict(container.labels) if container.labels else {} + + # Match by ID or ID prefix + if container.id == identity or container.id.startswith(identity): + channel.close() + return self._container_to_workload(container, labels) + + # Match by hostname (short ID) + if container.id[:12] == identity: + channel.close() + return self._container_to_workload(container, labels) + + # Match by name label (nerdctl style) + if labels.get("nerdctl/name") == identity: + channel.close() + return self._container_to_workload(container, labels) + + channel.close() + return None + except Exception as e: + print(f"[containerd] Failed to get container {identity}: {e}") + return None + + def _container_to_workload(self, container, labels: dict) -> Optional[Workload]: + """Convert containerd container to Workload.""" + try: + # Get network info from CNI state + networks, ips = self._get_cni_networks(container.id) + + # Container name (from nerdctl label or short ID) + name = labels.get("nerdctl/name", container.id[:12]) + hostname = container.id[:12] + + # Get exposed ports from labels (nerdctl stores them as labels) + ports = [] + for key, value in labels.items(): + if key.startswith("nerdctl/ports/"): + # Format: nerdctl/ports/tcp/80 = 0.0.0.0:8080 + parts = key.split("/") + if len(parts) >= 4: + ports.append(parts[3]) + + # Build addresses in format {container_name}.{network_name} + # This is how containers are reachable in containerd/nerdctl + addresses = [] + for network in networks: + addresses.append(f"{name}.{network}") + + return Workload( + id=container.id, + name=name, + hostname=hostname, + runtime=Runtime.CONTAINERD.value, + workload_type=WorkloadType.CONTAINER.value, + node=None, + namespace=None, + addresses=addresses, + isolation_groups=networks, + ports=ports, + labels=labels, + ) + except Exception as e: + print(f"[containerd] Failed to convert container: {e}") + return None + + def _start_cni_watcher(self, callback: Callable[[str, Workload], None]): + """Start watching CNI state directory for network changes.""" + class CNIEventHandler(FileSystemEventHandler): + def __init__(handler_self, adapter): + handler_self.adapter = adapter + + def on_created(handler_self, event): + if event.is_directory: + return + handler_self._handle_cni_change(event.src_path) + + def on_deleted(handler_self, event): + if event.is_directory: + return + handler_self._handle_cni_change(event.src_path) + + def _handle_cni_change(handler_self, filepath): + # Extract container ID from filename + filename = os.path.basename(filepath) + parts = filename.split("-") + if len(parts) >= 2: + # Try to find the container ID part (usually second) + for part in parts[1:]: + if len(part) >= 12: + workload = handler_self.adapter._get_workload(part[:12]) + if workload: + callback(EventType.NETWORK_CHANGED, workload) + break + + self._cni_observer = Observer() + self._cni_observer.schedule( + CNIEventHandler(self), + str(self.cni_state_path), + recursive=False + ) + self._cni_observer.start() + print(f"[containerd] Started CNI state watcher on {self.cni_state_path}") + + def _get_cni_networks(self, container_id: str) -> tuple: + """ + Get networks and IPs from CNI state files. + + CNI stores results in files like: + {network}-{namespace}-{container_id}-{interface} + + Network names may contain hyphens (e.g., "discovery_team-a"). + Container IDs are 64-char hex strings, so we match on the ID pattern. + The namespace suffix (e.g., "-default", "-moby") is stripped. + + Returns (networks: list[str], ips: list[str]) + """ + networks = [] + ips = [] + + if not self.cni_state_path.exists(): + return networks, ips + + # CNI files contain container ID (full or prefix) + short_id = container_id[:12] + + for result_file in self.cni_state_path.glob(f"*{short_id}*"): + try: + with open(result_file) as f: + result = json.load(f) + + # Parse network name from filename + # Format: {network}-{namespace}-{container_id}-{interface} + # Container ID is 64 hex chars, interface is like "eth0" + # Split on container ID to get network name + filename = result_file.name + + # Find container ID position in filename + id_pos = filename.find(container_id[:12]) + if id_pos > 0: + # Network name is everything before the container ID minus trailing dash + network_name = filename[:id_pos].rstrip("-") + + # Strip namespace suffix (e.g., "-default", "-moby") + namespace_suffix = f"-{self.namespace}" + if network_name.endswith(namespace_suffix): + network_name = network_name[:-len(namespace_suffix)] + + if network_name: + networks.append(network_name) + + # Get IPs from CNI result + for ip_config in result.get("ips", []): + addr = ip_config.get("address", "").split("/")[0] + if addr: + ips.append(addr) + + except (json.JSONDecodeError, KeyError, IOError): + continue + + return list(set(networks)), list(set(ips)) diff --git a/discovery/watcher/runtimes/docker.py b/discovery/watcher/runtimes/docker.py new file mode 100644 index 000000000..40c4fc757 --- /dev/null +++ b/discovery/watcher/runtimes/docker.py @@ -0,0 +1,214 @@ +""" +Docker runtime adapter. + +Watches Docker daemon for container events and converts to unified Workload model. +""" + +import threading +from typing import Optional, Callable + +import docker +import docker.errors + +from models import Workload, Runtime, WorkloadType, EventType +from config import DockerConfig +from runtimes.interface import RuntimeAdapter + + +class DockerAdapter(RuntimeAdapter): + """ + Docker runtime adapter. + + Uses Docker SDK to watch for container events and list containers. + Network changes (connect/disconnect) are fully supported. + """ + + SOCKET_PATH = "/var/run/docker.sock" + + def __init__(self, config: DockerConfig): + """ + Initialize Docker adapter. + + Args: + config: DockerConfig instance + """ + # Extract socket path from URL if needed + socket = config.socket + if socket.startswith("unix://"): + socket = socket[7:] + + self.socket_path = socket or self.SOCKET_PATH + self.label_key = config.label_key + self.label_value = config.label_value + + self._client: Optional[docker.DockerClient] = None + self._stop_event = threading.Event() + + @property + def runtime_type(self) -> Runtime: + return Runtime.DOCKER + + def connect(self) -> bool: + """Connect to Docker daemon.""" + try: + self._client = docker.DockerClient(base_url=f"unix://{self.socket_path}") + self._client.ping() + return True + except Exception as e: + print(f"[docker] Failed to connect: {e}") + return False + + def close(self): + """Close Docker connection.""" + self._stop_event.set() + if self._client: + self._client.close() + self._client = None + + def list_workloads(self) -> list: + """List all discoverable containers (running, not paused).""" + if not self._client: + return [] + + # Build filter for discover label and running state + # Note: status=running excludes paused containers + filters = { + "label": f"{self.label_key}={self.label_value}", + "status": "running", + } + + workloads = [] + try: + for container in self._client.containers.list(filters=filters): + # Double-check container is not paused (status filter should handle this) + if container.status == "paused": + continue + workload = self._container_to_workload(container) + if workload: + workloads.append(workload) + except Exception as e: + print(f"[docker] Failed to list containers: {e}") + + return workloads + + def watch_events(self, callback: Callable[[str, Workload], None]) -> None: + """Watch Docker events and call callback for each.""" + if not self._client: + return + + self._stop_event.clear() + + try: + for event in self._client.events( + decode=True, + filters={"type": "container"} + ): + if self._stop_event.is_set(): + break + + action = event.get("Action") + container_id = event.get("id") + + if not container_id: + continue + + if action == "start": + workload = self._get_container_workload(container_id) + if workload: + callback(EventType.ADDED, workload) + + elif action in ("stop", "die", "kill"): + # Create minimal workload for deletion + attrs = event.get("Actor", {}).get("Attributes", {}) + workload = Workload( + id=container_id, + name=attrs.get("name", container_id[:12]), + hostname=container_id[:12], + runtime=Runtime.DOCKER.value, + workload_type=WorkloadType.CONTAINER.value, + ) + callback(EventType.DELETED, workload) + + elif action == "pause": + # Paused containers are not reachable - remove from discovery + attrs = event.get("Actor", {}).get("Attributes", {}) + workload = Workload( + id=container_id, + name=attrs.get("name", container_id[:12]), + hostname=container_id[:12], + runtime=Runtime.DOCKER.value, + workload_type=WorkloadType.CONTAINER.value, + ) + callback(EventType.PAUSED, workload) + + elif action == "unpause": + # Unpaused container is reachable again - re-add to discovery + workload = self._get_container_workload(container_id) + if workload: + callback(EventType.ADDED, workload) + + elif action in ("connect", "disconnect"): + # Network changed - re-fetch full workload + workload = self._get_container_workload(container_id) + if workload: + callback(EventType.NETWORK_CHANGED, workload) + + except Exception as e: + if not self._stop_event.is_set(): + print(f"[docker] Event watch error: {e}") + + def _get_container_workload(self, container_id: str) -> Optional[Workload]: + """Get workload for container ID, handling not found.""" + try: + container = self._client.containers.get(container_id) + return self._container_to_workload(container) + except docker.errors.NotFound: + return None + except Exception as e: + print(f"[docker] Failed to get container {container_id}: {e}") + return None + + def _container_to_workload(self, container) -> Optional[Workload]: + """Convert Docker container to Workload.""" + try: + attrs = container.attrs + labels = container.labels or {} + + # Skip if not discoverable + if labels.get(self.label_key) != self.label_value: + return None + + # Get network information + network_settings = attrs.get("NetworkSettings", {}) + networks_info = network_settings.get("Networks", {}) + + # Container name and hostname + name = container.name + hostname = container.id[:12] + + # Get exposed ports + exposed_ports = attrs.get("Config", {}).get("ExposedPorts", {}) + ports = [p.split("/")[0] for p in exposed_ports.keys()] if exposed_ports else [] + + # Build addresses in format {container_name}.{network_name} + # This is how containers are reachable in Docker networks + addresses = [] + for network_name in networks_info.keys(): + addresses.append(f"{name}.{network_name}") + + return Workload( + id=container.id, + name=name, + hostname=hostname, + runtime=Runtime.DOCKER.value, + workload_type=WorkloadType.CONTAINER.value, + node=None, # Docker single-node + namespace=None, + addresses=addresses, + isolation_groups=list(networks_info.keys()), + ports=ports, + labels=labels, + ) + except Exception as e: + print(f"[docker] Failed to convert container: {e}") + return None diff --git a/discovery/watcher/runtimes/interface.py b/discovery/watcher/runtimes/interface.py new file mode 100644 index 000000000..6f147a144 --- /dev/null +++ b/discovery/watcher/runtimes/interface.py @@ -0,0 +1,72 @@ +""" +Abstract runtime interface for container runtimes. + +All runtime adapters must implement this interface. +Runtime adapters are responsible for: +- Listing workloads from their runtime +- Watching for workload events +- Converting runtime-specific data to unified Workload model + +Runtime adapters do NOT handle reachability queries - that's the storage layer's job. +""" + +from abc import ABC, abstractmethod +from typing import Optional, Callable + +from models import Workload, Runtime, EventType + + +class RuntimeAdapter(ABC): + """ + Abstract interface for container runtime adapters. + + Each runtime (Docker, containerd, Kubernetes) implements this interface + to provide a unified way to discover and watch workloads. + """ + + @property + @abstractmethod + def runtime_type(self) -> Runtime: + """Return which runtime this adapter handles.""" + pass + + @abstractmethod + def connect(self) -> bool: + """ + Connect to the runtime. + Returns True if connected successfully. + """ + pass + + @abstractmethod + def close(self): + """Close connection to runtime.""" + pass + + # ==================== Discovery ==================== + + @abstractmethod + def list_workloads(self) -> list: + """ + List all discoverable workloads. + + Args: + label_selector: Optional dict of labels to filter by + + Returns: + List[Workload] of discoverable workloads + """ + pass + + @abstractmethod + def watch_events(self, callback: Callable[[EventType, Workload], None]) -> None: + """ + Watch for workload events and call callback for each. + + This should run in a loop until stopped. + + Args: + callback: Function called with (event_type, workload) for each event + event_type is one of the EventType enum values + """ + pass diff --git a/discovery/watcher/runtimes/kubernetes.py b/discovery/watcher/runtimes/kubernetes.py new file mode 100644 index 000000000..d70d7c935 --- /dev/null +++ b/discovery/watcher/runtimes/kubernetes.py @@ -0,0 +1,459 @@ +""" +Kubernetes runtime adapter. + +Watches Kubernetes API for Pod and Service events. +Evaluates NetworkPolicies for reachability (stored as metadata). +""" + +import threading +from typing import Optional, Callable + +from kubernetes import client, config, watch +from kubernetes.client.rest import ApiException + +from models import Workload, Runtime, WorkloadType, EventType +from config import KubernetesConfig +from runtimes.interface import RuntimeAdapter + + +class KubernetesAdapter(RuntimeAdapter): + """ + Kubernetes runtime adapter. + + Watches Pods and Services via Kubernetes API. + Supports both in-cluster and kubeconfig authentication. + Evaluates NetworkPolicies and stores as workload annotations. + """ + + def __init__(self, config: KubernetesConfig): + """ + Initialize Kubernetes adapter. + + Args: + config: KubernetesConfig instance + """ + self.namespace = config.namespace # None = all namespaces + self.label_key = config.label_key + self.label_value = config.label_value + self.include_services = config.watch_services + self.in_cluster = config.in_cluster + self.kubeconfig = config.kubeconfig + + self._v1: Optional[client.CoreV1Api] = None + self._networking: Optional[client.NetworkingV1Api] = None + self._stop_event = threading.Event() + self._resource_version = None + + @property + def runtime_type(self) -> Runtime: + return Runtime.KUBERNETES + + def connect(self) -> bool: + """Connect to Kubernetes API.""" + try: + # Load config based on settings + if self.in_cluster: + config.load_incluster_config() + print("[kubernetes] Using in-cluster config") + elif self.kubeconfig: + config.load_kube_config(config_file=self.kubeconfig) + print(f"[kubernetes] Using kubeconfig: {self.kubeconfig}") + else: + # Try in-cluster first, then kubeconfig + try: + config.load_incluster_config() + print("[kubernetes] Using in-cluster config") + except config.ConfigException: + config.load_kube_config() + print("[kubernetes] Using default kubeconfig") + + self._v1 = client.CoreV1Api() + self._networking = client.NetworkingV1Api() + + # Verify connection + self._v1.list_namespace(limit=1) + return True + + except Exception as e: + print(f"[kubernetes] Failed to connect: {e}") + return False + + def close(self): + """Close Kubernetes connection.""" + self._stop_event.set() + self._v1 = None + self._networking = None + + def list_workloads(self) -> list: + """List all discoverable Pods with their service addresses.""" + if not self._v1: + return [] + + workloads = [] + + # Build label selector string + selector_parts = [f"{self.label_key}={self.label_value}"] + selector_str = ",".join(selector_parts) + + # Cache all services for lookups (services selecting discoverable pods) + services_by_namespace = self._get_services_by_namespace() + + # List Pods + try: + if self.namespace: + pods = self._v1.list_namespaced_pod( + namespace=self.namespace, + label_selector=selector_str + ) + else: + pods = self._v1.list_pod_for_all_namespaces( + label_selector=selector_str + ) + + for pod in pods.items: + if pod.status.phase == "Running": + # Find services that select this pod + matching_services = self._find_services_for_pod( + pod, services_by_namespace.get(pod.metadata.namespace, []) + ) + workload = self._pod_to_workload(pod, matching_services) + if workload: + workloads.append(workload) + except Exception as e: + print(f"[kubernetes] Failed to list pods: {e}") + + return workloads + + def watch_events(self, callback: Callable[[str, Workload], None]) -> None: + """Watch Kubernetes Pod and Service events.""" + if not self._v1: + return + + self._stop_event.clear() + + # Watch Pods + pod_thread = threading.Thread( + target=self._watch_pods, + args=(callback,), + daemon=True + ) + pod_thread.start() + + # Watch Services (if enabled) + if self.include_services: + svc_thread = threading.Thread( + target=self._watch_services, + args=(callback,), + daemon=True + ) + svc_thread.start() + + # Keep main thread alive + while not self._stop_event.is_set(): + self._stop_event.wait(1) + + def _pod_to_workload(self, pod, services: list = None) -> Optional[Workload]: + """Convert Kubernetes Pod to Workload, including service addresses.""" + try: + labels = pod.metadata.labels or {} + namespace = pod.metadata.namespace + + # Build addresses list + addresses = [] + + # 1. Pod DNS: {pod-ip-dashed}.{namespace}.pod + if pod.status.pod_ip: + ip_dashed = pod.status.pod_ip.replace(".", "-") + addresses.append(f"{ip_dashed}.{namespace}.pod") + + # 2. Service DNS: {service-name}.{namespace}.svc for each service selecting this pod + if services: + for svc in services: + addresses.append(f"{svc.metadata.name}.{namespace}.svc") + + # Extract ports from all containers + ports = [] + for container in pod.spec.containers: + for port in (container.ports or []): + ports.append(str(port.container_port)) + + # Isolation groups: namespace + isolation_groups = [namespace] + + # Check for network policies affecting this pod + policy_info = self._get_pod_network_policies(pod) + + # Build service names annotation + service_names = [svc.metadata.name for svc in (services or [])] + + return Workload( + id=pod.metadata.uid, + name=pod.metadata.name, + hostname=pod.spec.hostname or pod.metadata.name, + runtime=Runtime.KUBERNETES.value, + workload_type=WorkloadType.POD.value, + node=pod.spec.node_name, + namespace=namespace, + addresses=addresses, + isolation_groups=isolation_groups, + ports=ports, + labels=labels, + annotations={ + **(pod.metadata.annotations or {}), + "network_policies": policy_info, + "services": ",".join(service_names) if service_names else "", + }, + ) + except Exception as e: + print(f"[kubernetes] Failed to convert pod: {e}") + return None + + def _get_services_by_namespace(self) -> dict: + """Get all services grouped by namespace.""" + services_by_ns = {} + try: + if self.namespace: + services = self._v1.list_namespaced_service(namespace=self.namespace) + else: + services = self._v1.list_service_for_all_namespaces() + + for svc in services.items: + ns = svc.metadata.namespace + if ns not in services_by_ns: + services_by_ns[ns] = [] + services_by_ns[ns].append(svc) + except Exception as e: + print(f"[kubernetes] Failed to list services: {e}") + + return services_by_ns + + def _find_services_for_pod(self, pod, services: list) -> list: + """Find services that select this pod.""" + matching = [] + pod_labels = pod.metadata.labels or {} + + for svc in services: + selector = svc.spec.selector + if not selector: + continue + + # Check if all selector labels match pod labels + if all(pod_labels.get(k) == v for k, v in selector.items()): + matching.append(svc) + + return matching + + def _service_to_workload(self, svc) -> Optional[Workload]: + """Convert Kubernetes Service to Workload (kept for compatibility).""" + """Convert Kubernetes Service to Workload.""" + try: + labels = svc.metadata.labels or {} + + # Build address in format {service_name}.{namespace}.svc + addresses = [f"{svc.metadata.name}.{svc.metadata.namespace}.svc"] + + # Extract ports from service spec + ports = [str(port.port) for port in (svc.spec.ports or [])] + + return Workload( + id=svc.metadata.uid, + name=svc.metadata.name, + hostname=svc.metadata.name, + runtime=Runtime.KUBERNETES.value, + workload_type=WorkloadType.SERVICE.value, + node=None, # Services are cluster-wide + namespace=svc.metadata.namespace, + addresses=addresses, + isolation_groups=[svc.metadata.namespace], + ports=ports, + labels=labels, + annotations=svc.metadata.annotations or {}, + ) + except Exception as e: + print(f"[kubernetes] Failed to convert service: {e}") + return None + + def _get_pod_network_policies(self, pod) -> str: + """Get NetworkPolicies affecting a pod.""" + if not self._networking: + return "unknown" + + try: + policies = self._networking.list_namespaced_network_policy( + namespace=pod.metadata.namespace + ) + + if not policies.items: + return "none (default allow)" + + # Find policies that apply to this pod + affecting = [] + for policy in policies.items: + selector = policy.spec.pod_selector + if self._selector_matches(selector, pod.metadata.labels or {}): + affecting.append(policy.metadata.name) + + if affecting: + return f"restricted by: {', '.join(affecting)}" + return "not targeted by any policy" + + except Exception as e: + return f"error: {e}" + + def _selector_matches(self, selector, labels: dict) -> bool: + """Check if label selector matches labels.""" + if not selector or not selector.match_labels: + return True # Empty selector matches all + + for key, value in selector.match_labels.items(): + if labels.get(key) != value: + return False + return True + + def _watch_pods(self, callback: Callable[[str, Workload], None]): + """Watch Pod events.""" + w = watch.Watch() + selector = f"{self.label_key}={self.label_value}" + + while not self._stop_event.is_set(): + try: + # Cache services for this watch cycle + services_by_ns = self._get_services_by_namespace() + + if self.namespace: + stream = w.stream( + self._v1.list_namespaced_pod, + namespace=self.namespace, + label_selector=selector, + resource_version=self._resource_version, + timeout_seconds=300, + ) + else: + stream = w.stream( + self._v1.list_pod_for_all_namespaces, + label_selector=selector, + resource_version=self._resource_version, + timeout_seconds=300, + ) + + for event in stream: + if self._stop_event.is_set(): + break + + event_type = event['type'] + pod = event['object'] + + # Update resource version for resume + self._resource_version = pod.metadata.resource_version + + # Find services that select this pod + matching_services = self._find_services_for_pod( + pod, services_by_ns.get(pod.metadata.namespace, []) + ) + + workload = self._pod_to_workload(pod, matching_services) + if not workload: + continue + + if event_type == "ADDED": + if pod.status.phase == "Running": + callback(EventType.ADDED, workload) + elif event_type == "MODIFIED": + if pod.status.phase == "Running": + callback(EventType.MODIFIED, workload) + elif pod.status.phase in ("Succeeded", "Failed", "Pending"): + # Pod is no longer running - remove from discovery + callback(EventType.DELETED, workload) + elif event_type == "DELETED": + callback(EventType.DELETED, workload) + + except ApiException as e: + if e.status == 410: # Gone - resource version too old + print("[kubernetes] Watch expired, restarting...") + self._resource_version = None + else: + print(f"[kubernetes] Pod watch error: {e}") + if not self._stop_event.is_set(): + self._stop_event.wait(5) + except Exception as e: + print(f"[kubernetes] Pod watch error: {e}") + if not self._stop_event.is_set(): + self._stop_event.wait(5) + + def _watch_services(self, callback: Callable[[str, Workload], None]): + """Watch Service events and update affected pods.""" + w = watch.Watch() + resource_version = None + selector = f"{self.label_key}={self.label_value}" + + while not self._stop_event.is_set(): + try: + # Watch ALL services (not just labeled ones) since they may select discoverable pods + if self.namespace: + stream = w.stream( + self._v1.list_namespaced_service, + namespace=self.namespace, + resource_version=resource_version, + timeout_seconds=300, + ) + else: + stream = w.stream( + self._v1.list_service_for_all_namespaces, + resource_version=resource_version, + timeout_seconds=300, + ) + + for event in stream: + if self._stop_event.is_set(): + break + + event_type = event['type'] + svc = event['object'] + resource_version = svc.metadata.resource_version + + # When a service changes, refresh all discoverable pods it might select + if event_type in ("ADDED", "MODIFIED", "DELETED"): + self._refresh_pods_for_service(svc, callback, selector) + + except ApiException as e: + if e.status == 410: + resource_version = None + else: + print(f"[kubernetes] Service watch error: {e}") + if not self._stop_event.is_set(): + self._stop_event.wait(5) + except Exception as e: + print(f"[kubernetes] Service watch error: {e}") + if not self._stop_event.is_set(): + self._stop_event.wait(5) + + def _refresh_pods_for_service(self, svc, callback, pod_selector: str): + """Refresh pods that a service selects.""" + if not svc.spec.selector: + return + + try: + # Get all services in this namespace (for full address list) + services_by_ns = self._get_services_by_namespace() + namespace_services = services_by_ns.get(svc.metadata.namespace, []) + + # Find discoverable pods in the same namespace + pods = self._v1.list_namespaced_pod( + namespace=svc.metadata.namespace, + label_selector=pod_selector + ) + + for pod in pods.items: + if pod.status.phase != "Running": + continue + + # Check if this service selects this pod + pod_labels = pod.metadata.labels or {} + if all(pod_labels.get(k) == v for k, v in svc.spec.selector.items()): + # Refresh this pod with updated service list + matching_services = self._find_services_for_pod(pod, namespace_services) + workload = self._pod_to_workload(pod, matching_services) + if workload: + callback(EventType.MODIFIED, workload) + except Exception as e: + print(f"[kubernetes] Failed to refresh pods for service {svc.metadata.name}: {e}") diff --git a/discovery/watcher/storage/__init__.py b/discovery/watcher/storage/__init__.py new file mode 100644 index 000000000..999326684 --- /dev/null +++ b/discovery/watcher/storage/__init__.py @@ -0,0 +1,21 @@ +"""Storage module for multi-runtime service discovery.""" + +from storage.interface import StorageInterface +from storage.etcd import EtcdStorage +from config import logger, Config + +def create_storage(config: Config) -> StorageInterface: + """ + Create storage backend from config. + + Currently only supports etcd. + """ + logger.info("Connecting to etcd at %s:%d", config.etcd.host, config.etcd.port) + + # Initialize etcd storage + storage = EtcdStorage(config=config.etcd) + storage.connect() + + logger.info("Storage initialized") + + return storage diff --git a/discovery/watcher/storage/etcd.py b/discovery/watcher/storage/etcd.py new file mode 100644 index 000000000..32db27631 --- /dev/null +++ b/discovery/watcher/storage/etcd.py @@ -0,0 +1,103 @@ +""" +etcd-based storage for workload watcher (write-only). + +Key structure: + /discovery/workloads/{id} → Workload JSON + +Uses native etcd3 library for proper gRPC communication. +""" + +from typing import Optional, Set + +import etcd3 + +from models import Workload +from config import EtcdConfig, logger +from storage.interface import StorageInterface + + +class EtcdStorage(StorageInterface): + """ + etcd storage for registering/deregistering workloads. + + This is write-only - the server handles reads. + Uses etcd3 library for native gRPC support. + """ + + def __init__(self, config: EtcdConfig): + self.host = config.host + self.port = config.port + self.workloads_prefix = config.workloads_prefix + self.username = config.username + self.password = config.password + self._client: Optional[etcd3.Etcd3Client] = None + self._connected = False + + @property + def client(self) -> etcd3.Etcd3Client: + if self._client is None: + self._client = etcd3.client( + host=self.host, + port=self.port, + user=self.username, + password=self.password, + ) + return self._client + + def connect(self) -> bool: + """Connect to etcd.""" + try: + # Test connection by getting cluster status + self.client.status() + self._connected = True + logger.info("Connected to etcd at %s:%d", self.host, self.port) + return True + except Exception as e: + logger.error("Failed to connect to etcd: %s", e) + return False + + def close(self): + """Close connection.""" + if self._client: + self._client.close() + self._client = None + self._connected = False + + def register(self, workload: Workload) -> bool: + """Store workload in etcd.""" + try: + key = f"{self.workloads_prefix}{workload.id}" + self.client.put(key, workload.to_json()) + logger.debug("Registered workload: %s", workload.name) + return True + except Exception as e: + logger.error("Failed to register %s: %s", workload.name, e) + return False + + def deregister(self, workload_id: str) -> bool: + """Remove workload from etcd.""" + try: + key = f"{self.workloads_prefix}{workload_id}" + self.client.delete(key) + logger.debug("Deregistered workload: %s", workload_id[:12]) + return True + except Exception as e: + logger.error("Failed to deregister %s: %s", workload_id[:12], e) + return False + + def list_workload_ids(self) -> Set[str]: + """List all registered workload IDs (keys only, no values).""" + try: + ids = set() + # Use keys_only=True to avoid fetching values + for _, metadata in self.client.get_prefix(self.workloads_prefix, keys_only=True): + # Extract ID from key: /discovery/workloads/{id} + key = metadata.key.decode("utf-8") + workload_id = key[len(self.workloads_prefix):] + if workload_id: + ids.add(workload_id) + logger.debug("Listed %d workload IDs from etcd", len(ids)) + return ids + except Exception as e: + logger.error("Failed to list workload IDs: %s", e) + return set() diff --git a/discovery/watcher/storage/interface.py b/discovery/watcher/storage/interface.py new file mode 100644 index 000000000..a0097a4e6 --- /dev/null +++ b/discovery/watcher/storage/interface.py @@ -0,0 +1,57 @@ +""" +Storage interface for workload watcher (write-only). + +The watcher only needs to register/deregister workloads to etcd. +""" + +from abc import ABC, abstractmethod +from typing import Set +from models import Workload + + +class StorageInterface(ABC): + """ + Abstract interface for workload storage (write operations only). + + The watcher registers workloads discovered from runtimes. + """ + + @abstractmethod + def connect(self) -> bool: + """ + Connect to storage backend. + Returns True if connected successfully. + """ + pass + + @abstractmethod + def close(self): + """Close connection to storage backend.""" + pass + + @abstractmethod + def register(self, workload: Workload) -> bool: + """ + Register or update a workload. + Idempotent - overwrites if exists. + Returns True on success. + """ + pass + + @abstractmethod + def deregister(self, workload_id: str) -> bool: + """ + Remove a workload by ID. + Idempotent - no error if not exists. + Returns True on success. + """ + pass + + @abstractmethod + def list_workload_ids(self) -> Set[str]: + """ + List all registered workload IDs. + Used for reconciliation on startup. + Returns set of workload IDs. + """ + pass diff --git a/install/docker/apiserver.env b/install/docker/apiserver.env index 9c7cb6e33..320c67ade 100644 --- a/install/docker/apiserver.env +++ b/install/docker/apiserver.env @@ -28,4 +28,5 @@ DIRECTORY_SERVER_RATELIMIT_PER_CLIENT_RPS=100 DIRECTORY_SERVER_RATELIMIT_PER_CLIENT_BURST=200 DIRECTORY_LOGGER_LOG_LEVEL=DEBUG DIRECTORY_LOGGER_LOG_FORMAT=json -DIRECTORY_SERVER_LOGGING_VERBOSE=false \ No newline at end of file +DIRECTORY_SERVER_LOGGING_VERBOSE=false +DIRECTORY_SERVER_OASF_API_VALIDATION_DISABLE=true \ No newline at end of file diff --git a/proto/agntcy/dir/runtime/v1/discovery_service.proto b/proto/agntcy/dir/runtime/v1/discovery_service.proto new file mode 100644 index 000000000..6d4618017 --- /dev/null +++ b/proto/agntcy/dir/runtime/v1/discovery_service.proto @@ -0,0 +1,19 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package agntcy.dir.runtime.v1; + +import "agntcy/dir/runtime/v1/process.proto"; + +service DiscoveryService { + // List all record processes based on filters. + rpc ListProcesses(ListProcessesRequest) returns (stream Process); +} + +message ListProcessesRequest { + // Filters to apply when listing processes. + // Accepts regexp pattern if single value is provided. + repeated string filters = 1; +} diff --git a/proto/agntcy/dir/runtime/v1/process.proto b/proto/agntcy/dir/runtime/v1/process.proto new file mode 100644 index 000000000..5c1feb4c9 --- /dev/null +++ b/proto/agntcy/dir/runtime/v1/process.proto @@ -0,0 +1,26 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package agntcy.dir.runtime.v1; + +import "agntcy/dir/core/v1/record.proto"; + +// Information about a agent process instance. +message Process { + // Process ID + string pid = 1; + + // Name of the runtime environment + string runtime = 2; + + // Process creation timestamp in the RFC3339 format. + string created_at = 3; + + // Process metadata + map annotations = 4; + + // The record from which this process was created. + core.v1.RecordMeta record = 5; +} diff --git a/proto/buf.gen.yaml b/proto/buf.gen.yaml index d35a86c9e..d5c1178b2 100644 --- a/proto/buf.gen.yaml +++ b/proto/buf.gen.yaml @@ -29,6 +29,9 @@ managed: - path: agntcy/dir/naming/v1 file_option: go_package value: github.com/agntcy/dir/api/naming/v1 + - path: agntcy/dir/runtime/v1 + file_option: go_package + value: github.com/agntcy/dir/api/runtime/v1 plugins: # Stubs for Golang - remote: buf.build/protocolbuffers/go:v1.36.5 diff --git a/sdk/dir-js/src/models/agntcy/dir/runtime/v1/discovery_service_pb.d.ts b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/discovery_service_pb.d.ts new file mode 100644 index 000000000..b49536d1d --- /dev/null +++ b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/discovery_service_pb.d.ts @@ -0,0 +1,51 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +// @generated by protoc-gen-es v2.9.0 with parameter "import_extension=js" +// @generated from file agntcy/dir/runtime/v1/discovery_service.proto (package agntcy.dir.runtime.v1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; +import type { ProcessSchema } from "./process_pb.js"; + +/** + * Describes the file agntcy/dir/runtime/v1/discovery_service.proto. + */ +export declare const file_agntcy_dir_runtime_v1_discovery_service: GenFile; + +/** + * @generated from message agntcy.dir.runtime.v1.ListProcessesRequest + */ +export declare type ListProcessesRequest = Message<"agntcy.dir.runtime.v1.ListProcessesRequest"> & { + /** + * Filters to apply when listing processes. + * Accepts regexp pattern if single value is provided. + * + * @generated from field: repeated string filters = 1; + */ + filters: string[]; +}; + +/** + * Describes the message agntcy.dir.runtime.v1.ListProcessesRequest. + * Use `create(ListProcessesRequestSchema)` to create a new message. + */ +export declare const ListProcessesRequestSchema: GenMessage; + +/** + * @generated from service agntcy.dir.runtime.v1.DiscoveryService + */ +export declare const DiscoveryService: GenService<{ + /** + * List all record processes based on filters. + * + * @generated from rpc agntcy.dir.runtime.v1.DiscoveryService.ListProcesses + */ + listProcesses: { + methodKind: "server_streaming"; + input: typeof ListProcessesRequestSchema; + output: typeof ProcessSchema; + }, +}>; + diff --git a/sdk/dir-js/src/models/agntcy/dir/runtime/v1/discovery_service_pb.js b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/discovery_service_pb.js new file mode 100644 index 000000000..22964c2b8 --- /dev/null +++ b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/discovery_service_pb.js @@ -0,0 +1,29 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +// @generated by protoc-gen-es v2.9.0 with parameter "import_extension=js" +// @generated from file agntcy/dir/runtime/v1/discovery_service.proto (package agntcy.dir.runtime.v1, syntax proto3) +/* eslint-disable */ + +import { fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv2"; +import { file_agntcy_dir_runtime_v1_process } from "./process_pb.js"; + +/** + * Describes the file agntcy/dir/runtime/v1/discovery_service.proto. + */ +export const file_agntcy_dir_runtime_v1_discovery_service = /*@__PURE__*/ + fileDesc("Ci1hZ250Y3kvZGlyL3J1bnRpbWUvdjEvZGlzY292ZXJ5X3NlcnZpY2UucHJvdG8SFWFnbnRjeS5kaXIucnVudGltZS52MSInChRMaXN0UHJvY2Vzc2VzUmVxdWVzdBIPCgdmaWx0ZXJzGAEgAygJMnIKEERpc2NvdmVyeVNlcnZpY2USXgoNTGlzdFByb2Nlc3NlcxIrLmFnbnRjeS5kaXIucnVudGltZS52MS5MaXN0UHJvY2Vzc2VzUmVxdWVzdBoeLmFnbnRjeS5kaXIucnVudGltZS52MS5Qcm9jZXNzMAFCzwEKGWNvbS5hZ250Y3kuZGlyLnJ1bnRpbWUudjFCFURpc2NvdmVyeVNlcnZpY2VQcm90b1ABWiRnaXRodWIuY29tL2FnbnRjeS9kaXIvYXBpL3J1bnRpbWUvdjGiAgNBRFKqAhVBZ250Y3kuRGlyLlJ1bnRpbWUuVjHKAhVBZ250Y3lcRGlyXFJ1bnRpbWVcVjHiAiFBZ250Y3lcRGlyXFJ1bnRpbWVcVjFcR1BCTWV0YWRhdGHqAhhBZ250Y3k6OkRpcjo6UnVudGltZTo6VjFiBnByb3RvMw", [file_agntcy_dir_runtime_v1_process]); + +/** + * Describes the message agntcy.dir.runtime.v1.ListProcessesRequest. + * Use `create(ListProcessesRequestSchema)` to create a new message. + */ +export const ListProcessesRequestSchema = /*@__PURE__*/ + messageDesc(file_agntcy_dir_runtime_v1_discovery_service, 0); + +/** + * @generated from service agntcy.dir.runtime.v1.DiscoveryService + */ +export const DiscoveryService = /*@__PURE__*/ + serviceDesc(file_agntcy_dir_runtime_v1_discovery_service, 0); + diff --git a/sdk/dir-js/src/models/agntcy/dir/runtime/v1/process_pb.d.ts b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/process_pb.d.ts new file mode 100644 index 000000000..a0cb713b1 --- /dev/null +++ b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/process_pb.d.ts @@ -0,0 +1,64 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +// @generated by protoc-gen-es v2.9.0 with parameter "import_extension=js" +// @generated from file agntcy/dir/runtime/v1/process.proto (package agntcy.dir.runtime.v1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; +import type { RecordMeta } from "../../core/v1/record_pb.js"; + +/** + * Describes the file agntcy/dir/runtime/v1/process.proto. + */ +export declare const file_agntcy_dir_runtime_v1_process: GenFile; + +/** + * Information about a agent process instance. + * + * @generated from message agntcy.dir.runtime.v1.Process + */ +export declare type Process = Message<"agntcy.dir.runtime.v1.Process"> & { + /** + * Process ID + * + * @generated from field: string pid = 1; + */ + pid: string; + + /** + * Name of the runtime environment + * + * @generated from field: string runtime = 2; + */ + runtime: string; + + /** + * Process creation timestamp in the RFC3339 format. + * + * @generated from field: string created_at = 3; + */ + createdAt: string; + + /** + * Process metadata + * + * @generated from field: map annotations = 4; + */ + annotations: { [key: string]: string }; + + /** + * The record from which this process was created. + * + * @generated from field: agntcy.dir.core.v1.RecordMeta record = 5; + */ + record?: RecordMeta; +}; + +/** + * Describes the message agntcy.dir.runtime.v1.Process. + * Use `create(ProcessSchema)` to create a new message. + */ +export declare const ProcessSchema: GenMessage; + diff --git a/sdk/dir-js/src/models/agntcy/dir/runtime/v1/process_pb.js b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/process_pb.js new file mode 100644 index 000000000..615ab7e65 --- /dev/null +++ b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/process_pb.js @@ -0,0 +1,23 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +// @generated by protoc-gen-es v2.9.0 with parameter "import_extension=js" +// @generated from file agntcy/dir/runtime/v1/process.proto (package agntcy.dir.runtime.v1, syntax proto3) +/* eslint-disable */ + +import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2"; +import { file_agntcy_dir_core_v1_record } from "../../core/v1/record_pb.js"; + +/** + * Describes the file agntcy/dir/runtime/v1/process.proto. + */ +export const file_agntcy_dir_runtime_v1_process = /*@__PURE__*/ + fileDesc("CiNhZ250Y3kvZGlyL3J1bnRpbWUvdjEvcHJvY2Vzcy5wcm90bxIVYWdudGN5LmRpci5ydW50aW1lLnYxIuUBCgdQcm9jZXNzEgsKA3BpZBgBIAEoCRIPCgdydW50aW1lGAIgASgJEhIKCmNyZWF0ZWRfYXQYAyABKAkSRAoLYW5ub3RhdGlvbnMYBCADKAsyLy5hZ250Y3kuZGlyLnJ1bnRpbWUudjEuUHJvY2Vzcy5Bbm5vdGF0aW9uc0VudHJ5Ei4KBnJlY29yZBgFIAEoCzIeLmFnbnRjeS5kaXIuY29yZS52MS5SZWNvcmRNZXRhGjIKEEFubm90YXRpb25zRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4AULGAQoZY29tLmFnbnRjeS5kaXIucnVudGltZS52MUIMUHJvY2Vzc1Byb3RvUAFaJGdpdGh1Yi5jb20vYWdudGN5L2Rpci9hcGkvcnVudGltZS92MaICA0FEUqoCFUFnbnRjeS5EaXIuUnVudGltZS5WMcoCFUFnbnRjeVxEaXJcUnVudGltZVxWMeICIUFnbnRjeVxEaXJcUnVudGltZVxWMVxHUEJNZXRhZGF0YeoCGEFnbnRjeTo6RGlyOjpSdW50aW1lOjpWMWIGcHJvdG8z", [file_agntcy_dir_core_v1_record]); + +/** + * Describes the message agntcy.dir.runtime.v1.Process. + * Use `create(ProcessSchema)` to create a new message. + */ +export const ProcessSchema = /*@__PURE__*/ + messageDesc(file_agntcy_dir_runtime_v1_process, 0); + diff --git a/sdk/dir-js/src/models/agntcy/dir/runtime/v1/runtime_service_pb.d.ts b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/runtime_service_pb.d.ts new file mode 100644 index 000000000..5506dd6bf --- /dev/null +++ b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/runtime_service_pb.d.ts @@ -0,0 +1,51 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +// @generated by protoc-gen-es v2.9.0 with parameter "import_extension=js" +// @generated from file agntcy/dir/runtime/v1/runtime_service.proto (package agntcy.dir.runtime.v1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; +import type { ProcessSchema } from "./process_pb.js"; + +/** + * Describes the file agntcy/dir/runtime/v1/runtime_service.proto. + */ +export declare const file_agntcy_dir_runtime_v1_runtime_service: GenFile; + +/** + * @generated from message agntcy.dir.runtime.v1.ListProcessesRequest + */ +export declare type ListProcessesRequest = Message<"agntcy.dir.runtime.v1.ListProcessesRequest"> & { + /** + * Filters to apply when listing processes. + * Accepts regexp pattern if single value is provided. + * + * @generated from field: repeated string filters = 1; + */ + filters: string[]; +}; + +/** + * Describes the message agntcy.dir.runtime.v1.ListProcessesRequest. + * Use `create(ListProcessesRequestSchema)` to create a new message. + */ +export declare const ListProcessesRequestSchema: GenMessage; + +/** + * @generated from service agntcy.dir.runtime.v1.RuntimeService + */ +export declare const RuntimeService: GenService<{ + /** + * List all record processes based on filters. + * + * @generated from rpc agntcy.dir.runtime.v1.RuntimeService.ListProcesses + */ + listProcesses: { + methodKind: "server_streaming"; + input: typeof ListProcessesRequestSchema; + output: typeof ProcessSchema; + }, +}>; + diff --git a/sdk/dir-js/src/models/agntcy/dir/runtime/v1/runtime_service_pb.js b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/runtime_service_pb.js new file mode 100644 index 000000000..b412b6803 --- /dev/null +++ b/sdk/dir-js/src/models/agntcy/dir/runtime/v1/runtime_service_pb.js @@ -0,0 +1,29 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +// @generated by protoc-gen-es v2.9.0 with parameter "import_extension=js" +// @generated from file agntcy/dir/runtime/v1/runtime_service.proto (package agntcy.dir.runtime.v1, syntax proto3) +/* eslint-disable */ + +import { fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv2"; +import { file_agntcy_dir_runtime_v1_process } from "./process_pb.js"; + +/** + * Describes the file agntcy/dir/runtime/v1/runtime_service.proto. + */ +export const file_agntcy_dir_runtime_v1_runtime_service = /*@__PURE__*/ + fileDesc("CithZ250Y3kvZGlyL3J1bnRpbWUvdjEvcnVudGltZV9zZXJ2aWNlLnByb3RvEhVhZ250Y3kuZGlyLnJ1bnRpbWUudjEiJwoUTGlzdFByb2Nlc3Nlc1JlcXVlc3QSDwoHZmlsdGVycxgBIAMoCTJwCg5SdW50aW1lU2VydmljZRJeCg1MaXN0UHJvY2Vzc2VzEisuYWdudGN5LmRpci5ydW50aW1lLnYxLkxpc3RQcm9jZXNzZXNSZXF1ZXN0Gh4uYWdudGN5LmRpci5ydW50aW1lLnYxLlByb2Nlc3MwAULNAQoZY29tLmFnbnRjeS5kaXIucnVudGltZS52MUITUnVudGltZVNlcnZpY2VQcm90b1ABWiRnaXRodWIuY29tL2FnbnRjeS9kaXIvYXBpL3J1bnRpbWUvdjGiAgNBRFKqAhVBZ250Y3kuRGlyLlJ1bnRpbWUuVjHKAhVBZ250Y3lcRGlyXFJ1bnRpbWVcVjHiAiFBZ250Y3lcRGlyXFJ1bnRpbWVcVjFcR1BCTWV0YWRhdGHqAhhBZ250Y3k6OkRpcjo6UnVudGltZTo6VjFiBnByb3RvMw", [file_agntcy_dir_runtime_v1_process]); + +/** + * Describes the message agntcy.dir.runtime.v1.ListProcessesRequest. + * Use `create(ListProcessesRequestSchema)` to create a new message. + */ +export const ListProcessesRequestSchema = /*@__PURE__*/ + messageDesc(file_agntcy_dir_runtime_v1_runtime_service, 0); + +/** + * @generated from service agntcy.dir.runtime.v1.RuntimeService + */ +export const RuntimeService = /*@__PURE__*/ + serviceDesc(file_agntcy_dir_runtime_v1_runtime_service, 0); + diff --git a/sdk/dir-py/agntcy/dir/naming/v1/domain_verification_pb2.py b/sdk/dir-py/agntcy/dir/naming/v1/domain_verification_pb2.py deleted file mode 100644 index d77005b82..000000000 --- a/sdk/dir-py/agntcy/dir/naming/v1/domain_verification_pb2.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# NO CHECKED-IN PROTOBUF GENCODE -# source: agntcy/dir/naming/v1/domain_verification.proto -# Protobuf Python Version: 6.32.1 -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import runtime_version as _runtime_version -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -_runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 6, - 32, - 1, - '', - 'agntcy/dir/naming/v1/domain_verification.proto' -) -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.agntcy/dir/naming/v1/domain_verification.proto\x12\x14\x61gntcy.dir.naming.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\xa7\x01\n\x12\x44omainVerification\x12\x16\n\x06\x64omain\x18\x01 \x01(\tR\x06\x64omain\x12\x16\n\x06method\x18\x02 \x01(\tR\x06method\x12$\n\x0ematched_key_id\x18\x03 \x01(\tR\x0cmatchedKeyId\x12;\n\x0bverified_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\nverifiedAtB\xcb\x01\n\x18\x63om.agntcy.dir.naming.v1B\x17\x44omainVerificationProtoP\x01Z#github.com/agntcy/dir/api/naming/v1\xa2\x02\x03\x41\x44N\xaa\x02\x14\x41gntcy.Dir.Naming.V1\xca\x02\x14\x41gntcy\\Dir\\Naming\\V1\xe2\x02 Agntcy\\Dir\\Naming\\V1\\GPBMetadata\xea\x02\x17\x41gntcy::Dir::Naming::V1b\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'agntcy.dir.naming.v1.domain_verification_pb2', _globals) -if not _descriptor._USE_C_DESCRIPTORS: - _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'\n\030com.agntcy.dir.naming.v1B\027DomainVerificationProtoP\001Z#github.com/agntcy/dir/api/naming/v1\242\002\003ADN\252\002\024Agntcy.Dir.Naming.V1\312\002\024Agntcy\\Dir\\Naming\\V1\342\002 Agntcy\\Dir\\Naming\\V1\\GPBMetadata\352\002\027Agntcy::Dir::Naming::V1' - _globals['_DOMAINVERIFICATION']._serialized_start=106 - _globals['_DOMAINVERIFICATION']._serialized_end=273 -# @@protoc_insertion_point(module_scope) diff --git a/sdk/dir-py/agntcy/dir/naming/v1/domain_verification_pb2.pyi b/sdk/dir-py/agntcy/dir/naming/v1/domain_verification_pb2.pyi deleted file mode 100644 index c97e65d61..000000000 --- a/sdk/dir-py/agntcy/dir/naming/v1/domain_verification_pb2.pyi +++ /dev/null @@ -1,18 +0,0 @@ -from google.protobuf import timestamp_pb2 as _timestamp_pb2 -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union - -DESCRIPTOR: _descriptor.FileDescriptor - -class DomainVerification(_message.Message): - __slots__ = ("domain", "method", "matched_key_id", "verified_at") - DOMAIN_FIELD_NUMBER: _ClassVar[int] - METHOD_FIELD_NUMBER: _ClassVar[int] - MATCHED_KEY_ID_FIELD_NUMBER: _ClassVar[int] - VERIFIED_AT_FIELD_NUMBER: _ClassVar[int] - domain: str - method: str - matched_key_id: str - verified_at: _timestamp_pb2.Timestamp - def __init__(self, domain: _Optional[str] = ..., method: _Optional[str] = ..., matched_key_id: _Optional[str] = ..., verified_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ... diff --git a/sdk/dir-py/agntcy/dir/runtime/v1/discovery_service_pb2.py b/sdk/dir-py/agntcy/dir/runtime/v1/discovery_service_pb2.py new file mode 100644 index 000000000..6481edce3 --- /dev/null +++ b/sdk/dir-py/agntcy/dir/runtime/v1/discovery_service_pb2.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: agntcy/dir/runtime/v1/discovery_service.proto +# Protobuf Python Version: 6.32.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 32, + 1, + '', + 'agntcy/dir/runtime/v1/discovery_service.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from agntcy.dir.runtime.v1 import process_pb2 as agntcy_dot_dir_dot_runtime_dot_v1_dot_process__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n-agntcy/dir/runtime/v1/discovery_service.proto\x12\x15\x61gntcy.dir.runtime.v1\x1a#agntcy/dir/runtime/v1/process.proto\"0\n\x14ListProcessesRequest\x12\x18\n\x07\x66ilters\x18\x01 \x03(\tR\x07\x66ilters2r\n\x10\x44iscoveryService\x12^\n\rListProcesses\x12+.agntcy.dir.runtime.v1.ListProcessesRequest\x1a\x1e.agntcy.dir.runtime.v1.Process0\x01\x42\xcf\x01\n\x19\x63om.agntcy.dir.runtime.v1B\x15\x44iscoveryServiceProtoP\x01Z$github.com/agntcy/dir/api/runtime/v1\xa2\x02\x03\x41\x44R\xaa\x02\x15\x41gntcy.Dir.Runtime.V1\xca\x02\x15\x41gntcy\\Dir\\Runtime\\V1\xe2\x02!Agntcy\\Dir\\Runtime\\V1\\GPBMetadata\xea\x02\x18\x41gntcy::Dir::Runtime::V1b\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'agntcy.dir.runtime.v1.discovery_service_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\031com.agntcy.dir.runtime.v1B\025DiscoveryServiceProtoP\001Z$github.com/agntcy/dir/api/runtime/v1\242\002\003ADR\252\002\025Agntcy.Dir.Runtime.V1\312\002\025Agntcy\\Dir\\Runtime\\V1\342\002!Agntcy\\Dir\\Runtime\\V1\\GPBMetadata\352\002\030Agntcy::Dir::Runtime::V1' + _globals['_LISTPROCESSESREQUEST']._serialized_start=109 + _globals['_LISTPROCESSESREQUEST']._serialized_end=157 + _globals['_DISCOVERYSERVICE']._serialized_start=159 + _globals['_DISCOVERYSERVICE']._serialized_end=273 +# @@protoc_insertion_point(module_scope) diff --git a/sdk/dir-py/agntcy/dir/runtime/v1/discovery_service_pb2.pyi b/sdk/dir-py/agntcy/dir/runtime/v1/discovery_service_pb2.pyi new file mode 100644 index 000000000..9b2ef8939 --- /dev/null +++ b/sdk/dir-py/agntcy/dir/runtime/v1/discovery_service_pb2.pyi @@ -0,0 +1,13 @@ +from agntcy.dir.runtime.v1 import process_pb2 as _process_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Iterable as _Iterable, Optional as _Optional + +DESCRIPTOR: _descriptor.FileDescriptor + +class ListProcessesRequest(_message.Message): + __slots__ = ("filters",) + FILTERS_FIELD_NUMBER: _ClassVar[int] + filters: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, filters: _Optional[_Iterable[str]] = ...) -> None: ... diff --git a/sdk/dir-py/agntcy/dir/runtime/v1/discovery_service_pb2_grpc.py b/sdk/dir-py/agntcy/dir/runtime/v1/discovery_service_pb2_grpc.py new file mode 100644 index 000000000..160fb4689 --- /dev/null +++ b/sdk/dir-py/agntcy/dir/runtime/v1/discovery_service_pb2_grpc.py @@ -0,0 +1,79 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from agntcy.dir.runtime.v1 import discovery_service_pb2 as agntcy_dot_dir_dot_runtime_dot_v1_dot_discovery__service__pb2 +from agntcy.dir.runtime.v1 import process_pb2 as agntcy_dot_dir_dot_runtime_dot_v1_dot_process__pb2 + + +class DiscoveryServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ListProcesses = channel.unary_stream( + '/agntcy.dir.runtime.v1.DiscoveryService/ListProcesses', + request_serializer=agntcy_dot_dir_dot_runtime_dot_v1_dot_discovery__service__pb2.ListProcessesRequest.SerializeToString, + response_deserializer=agntcy_dot_dir_dot_runtime_dot_v1_dot_process__pb2.Process.FromString, + _registered_method=True) + + +class DiscoveryServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def ListProcesses(self, request, context): + """List all record processes based on filters. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_DiscoveryServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ListProcesses': grpc.unary_stream_rpc_method_handler( + servicer.ListProcesses, + request_deserializer=agntcy_dot_dir_dot_runtime_dot_v1_dot_discovery__service__pb2.ListProcessesRequest.FromString, + response_serializer=agntcy_dot_dir_dot_runtime_dot_v1_dot_process__pb2.Process.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'agntcy.dir.runtime.v1.DiscoveryService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('agntcy.dir.runtime.v1.DiscoveryService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class DiscoveryService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def ListProcesses(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/agntcy.dir.runtime.v1.DiscoveryService/ListProcesses', + agntcy_dot_dir_dot_runtime_dot_v1_dot_discovery__service__pb2.ListProcessesRequest.SerializeToString, + agntcy_dot_dir_dot_runtime_dot_v1_dot_process__pb2.Process.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/sdk/dir-py/agntcy/dir/runtime/v1/process_pb2.py b/sdk/dir-py/agntcy/dir/runtime/v1/process_pb2.py new file mode 100644 index 000000000..80ec40ea1 --- /dev/null +++ b/sdk/dir-py/agntcy/dir/runtime/v1/process_pb2.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: agntcy/dir/runtime/v1/process.proto +# Protobuf Python Version: 6.32.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 32, + 1, + '', + 'agntcy/dir/runtime/v1/process.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from agntcy.dir.core.v1 import record_pb2 as agntcy_dot_dir_dot_core_dot_v1_dot_record__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#agntcy/dir/runtime/v1/process.proto\x12\x15\x61gntcy.dir.runtime.v1\x1a\x1f\x61gntcy/dir/core/v1/record.proto\"\x9f\x02\n\x07Process\x12\x10\n\x03pid\x18\x01 \x01(\tR\x03pid\x12\x18\n\x07runtime\x18\x02 \x01(\tR\x07runtime\x12\x1d\n\ncreated_at\x18\x03 \x01(\tR\tcreatedAt\x12Q\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32/.agntcy.dir.runtime.v1.Process.AnnotationsEntryR\x0b\x61nnotations\x12\x36\n\x06record\x18\x05 \x01(\x0b\x32\x1e.agntcy.dir.core.v1.RecordMetaR\x06record\x1a>\n\x10\x41nnotationsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\xc6\x01\n\x19\x63om.agntcy.dir.runtime.v1B\x0cProcessProtoP\x01Z$github.com/agntcy/dir/api/runtime/v1\xa2\x02\x03\x41\x44R\xaa\x02\x15\x41gntcy.Dir.Runtime.V1\xca\x02\x15\x41gntcy\\Dir\\Runtime\\V1\xe2\x02!Agntcy\\Dir\\Runtime\\V1\\GPBMetadata\xea\x02\x18\x41gntcy::Dir::Runtime::V1b\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'agntcy.dir.runtime.v1.process_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\031com.agntcy.dir.runtime.v1B\014ProcessProtoP\001Z$github.com/agntcy/dir/api/runtime/v1\242\002\003ADR\252\002\025Agntcy.Dir.Runtime.V1\312\002\025Agntcy\\Dir\\Runtime\\V1\342\002!Agntcy\\Dir\\Runtime\\V1\\GPBMetadata\352\002\030Agntcy::Dir::Runtime::V1' + _globals['_PROCESS_ANNOTATIONSENTRY']._loaded_options = None + _globals['_PROCESS_ANNOTATIONSENTRY']._serialized_options = b'8\001' + _globals['_PROCESS']._serialized_start=96 + _globals['_PROCESS']._serialized_end=383 + _globals['_PROCESS_ANNOTATIONSENTRY']._serialized_start=321 + _globals['_PROCESS_ANNOTATIONSENTRY']._serialized_end=383 +# @@protoc_insertion_point(module_scope) diff --git a/sdk/dir-py/agntcy/dir/runtime/v1/process_pb2.pyi b/sdk/dir-py/agntcy/dir/runtime/v1/process_pb2.pyi new file mode 100644 index 000000000..007eae6d7 --- /dev/null +++ b/sdk/dir-py/agntcy/dir/runtime/v1/process_pb2.pyi @@ -0,0 +1,28 @@ +from agntcy.dir.core.v1 import record_pb2 as _record_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class Process(_message.Message): + __slots__ = ("pid", "runtime", "created_at", "annotations", "record") + class AnnotationsEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + PID_FIELD_NUMBER: _ClassVar[int] + RUNTIME_FIELD_NUMBER: _ClassVar[int] + CREATED_AT_FIELD_NUMBER: _ClassVar[int] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + RECORD_FIELD_NUMBER: _ClassVar[int] + pid: str + runtime: str + created_at: str + annotations: _containers.ScalarMap[str, str] + record: _record_pb2.RecordMeta + def __init__(self, pid: _Optional[str] = ..., runtime: _Optional[str] = ..., created_at: _Optional[str] = ..., annotations: _Optional[_Mapping[str, str]] = ..., record: _Optional[_Union[_record_pb2.RecordMeta, _Mapping]] = ...) -> None: ... diff --git a/sdk/dir-py/agntcy/dir/naming/v1/domain_verification_pb2_grpc.py b/sdk/dir-py/agntcy/dir/runtime/v1/process_pb2_grpc.py similarity index 100% rename from sdk/dir-py/agntcy/dir/naming/v1/domain_verification_pb2_grpc.py rename to sdk/dir-py/agntcy/dir/runtime/v1/process_pb2_grpc.py diff --git a/server/controller/runtime.go b/server/controller/runtime.go new file mode 100644 index 000000000..a9c79639c --- /dev/null +++ b/server/controller/runtime.go @@ -0,0 +1,26 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + runtimev1 "github.com/agntcy/dir/api/runtime/v1" + "github.com/agntcy/dir/server/types" +) + +type runtimeCtlr struct { + runtimev1.UnimplementedDiscoveryServiceServer + api types.DiscoveryAPI +} + +// NewRuntimeController creates a new runtime controller. +func NewRuntimeController(api types.DiscoveryAPI) runtimev1.DiscoveryServiceServer { + return &runtimeCtlr{ + UnimplementedDiscoveryServiceServer: runtimev1.UnimplementedDiscoveryServiceServer{}, + api: api, + } +} + +func (c *runtimeCtlr) ListProcesses(req *runtimev1.ListProcessesRequest, stream runtimev1.DiscoveryService_ListProcessesServer) error { + return c.api.ListProcesses(stream.Context(), req, stream.Send) +} diff --git a/server/go.mod b/server/go.mod index 2dd1a1d98..3df392ebb 100644 --- a/server/go.mod +++ b/server/go.mod @@ -20,6 +20,7 @@ require ( github.com/agntcy/dir/utils v0.6.1 github.com/agntcy/oasf-sdk/pkg v0.0.14 github.com/casbin/casbin/v2 v2.135.0 + github.com/containerd/containerd/v2 v2.2.1 github.com/glebarez/sqlite v1.11.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 @@ -44,9 +45,25 @@ require ( ) require ( + cyphar.com/go-pathrs v0.2.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/hcsshim v0.14.0-rc.1 // indirect github.com/ThalesIgnite/crypto11 v1.2.5 // indirect + github.com/containerd/cgroups/v3 v3.1.2 // indirect + github.com/containerd/containerd/api v1.10.0 // indirect + github.com/containerd/continuity v0.4.5 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/fifo v1.1.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v1.0.0-rc.2 // indirect + github.com/containerd/plugin v1.0.0 // indirect + github.com/containerd/ttrpc v1.2.7 // indirect + github.com/containerd/typeurl/v2 v2.2.3 // indirect + github.com/creack/pty v1.1.24 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/filecoin-project/go-clock v0.1.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-chi/chi/v5 v5.2.4 // indirect @@ -62,7 +79,9 @@ require ( github.com/go-openapi/swag/typeutils v0.25.4 // indirect github.com/go-openapi/swag/yamlutils v0.25.4 // indirect github.com/goccy/go-json v0.10.5 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/gnostic-models v0.7.1 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-github/v73 v73.0.0 // indirect github.com/google/go-querystring v1.2.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -72,9 +91,17 @@ require ( github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/miekg/pkcs11 v1.1.1 // indirect + github.com/moby/locker v1.0.1 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/signal v0.7.1 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/opencontainers/runtime-spec v1.3.0 // indirect + github.com/opencontainers/selinux v1.13.1 // indirect github.com/pion/transport/v4 v4.0.1 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/quic-go/quic-go v0.59.0 // indirect @@ -86,6 +113,8 @@ require ( github.com/tiendc/go-deepcopy v1.7.2 // indirect github.com/x448/float16 v0.8.4 // indirect gitlab.com/gitlab-org/api/client-go v1.14.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 // indirect @@ -154,7 +183,7 @@ require ( github.com/google/certificate-transparency-go v1.3.2 // indirect github.com/google/go-containerregistry v0.20.7 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a // indirect + github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4 // indirect diff --git a/server/go.sum b/server/go.sum index a639eb9cc..344da1a68 100644 --- a/server/go.sum +++ b/server/go.sum @@ -4,6 +4,7 @@ buf.build/gen/go/agntcy/oasf-sdk/protocolbuffers/go v1.36.11-20260115113053-9b11 buf.build/gen/go/agntcy/oasf-sdk/protocolbuffers/go v1.36.11-20260115113053-9b110d5996b7.1/go.mod h1:RA6Inven3rnt6tAMhFaQ7s2DfRWoyOyIQ9/WUX+29TQ= buf.build/gen/go/agntcy/oasf/protocolbuffers/go v1.36.11-20260109151047-d2be6d341048.1 h1:EeGnBiwWBhl7z7s6nKVtH+3HdHAWvV1JzqEcb365TA0= buf.build/gen/go/agntcy/oasf/protocolbuffers/go v1.36.11-20260109151047-d2be6d341048.1/go.mod h1:y7UzNChPK2OBXQxpwimQg6fSgSFLC1jZXTk9Y7HFfNc= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4= @@ -18,8 +19,12 @@ cloud.google.com/go/kms v1.23.2 h1:4IYDQL5hG4L+HzJBhzejUySoUOheh3Lk5YT4PCyyW6k= cloud.google.com/go/kms v1.23.2/go.mod h1:rZ5kK0I7Kn9W4erhYVoIRPtpizjunlrfU4fUkumUp8g= cloud.google.com/go/longrunning v0.7.0 h1:FV0+SYF1RIj59gyoWDRi45GiYUMM3K1qO51qoboQT1E= cloud.google.com/go/longrunning v0.7.0/go.mod h1:ySn2yXmjbK9Ba0zsQqunhDkYi0+9rlXIwnoAf+h+TPY= +cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8= +cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d h1:zjqpY4C7H15HjRPEenkS4SAn3Jy2eRRjkjZbGR30TOg= github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d/go.mod h1:XNqJ7hv2kY++g8XEHREpi+JqZo3+0l+CH2egBVN4yqM= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= @@ -45,6 +50,8 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.14.0-rc.1 h1:qAPXKwGOkVn8LlqgBN8GS0bxZ83hOJpcjxzmlQKxKsQ= +github.com/Microsoft/hcsshim v0.14.0-rc.1/go.mod h1:hTKFGbnDtQb1wHiOWv4v0eN+7boSWAHyK/tNAaYZL0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ThalesGroup/crypto11 v1.6.0 h1:Og9EMn44fBS4GNnGnH1aqHnF2wL6F7IU/RhpJajWX/4= github.com/ThalesGroup/crypto11 v1.6.0/go.mod h1:H6LRjN5R5SHxTrLqGNteisLDI0/IC6+SGx1pHtbwizE= @@ -103,16 +110,43 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= +github.com/containerd/cgroups/v3 v3.1.2 h1:OSosXMtkhI6Qove637tg1XgK4q+DhR0mX8Wi8EhrHa4= +github.com/containerd/cgroups/v3 v3.1.2/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= +github.com/containerd/containerd/api v1.10.0 h1:5n0oHYVBwN4VhoX9fFykCV9dF1/BvAXeg2F8W6UYq1o= +github.com/containerd/containerd/api v1.10.0/go.mod h1:NBm1OAk8ZL+LG8R0ceObGxT5hbUYj7CzTmR3xh0DlMM= +github.com/containerd/containerd/v2 v2.2.1 h1:TpyxcY4AL5A+07dxETevunVS5zxqzuq7ZqJXknM11yk= +github.com/containerd/containerd/v2 v2.2.1/go.mod h1:NR70yW1iDxe84F2iFWbR9xfAN0N2F0NcjTi1OVth4nU= +github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= +github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= +github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v1.0.0-rc.2 h1:0SPgaNZPVWGEi4grZdV8VRYQn78y+nm6acgLGv/QzE4= +github.com/containerd/platforms v1.0.0-rc.2/go.mod h1:J71L7B+aiM5SdIEqmd9wp6THLVRzJGXfNuWCZCllLA4= +github.com/containerd/plugin v1.0.0 h1:c8Kf1TNl6+e2TtMHZt+39yAPDbouRH9WAToRjex483Y= +github.com/containerd/plugin v1.0.0/go.mod h1:hQfJe5nmWfImiqT1q8Si3jLv3ynMUIBB47bQ+KexvO8= github.com/containerd/stargz-snapshotter/estargz v0.18.1 h1:cy2/lpgBXDA3cDKSyEfNOFMA/c10O1axL69EU7iirO8= github.com/containerd/stargz-snapshotter/estargz v0.18.1/go.mod h1:ALIEqa7B6oVDsrF37GkGN20SuvG/pIMm7FwP7ZmRb0Q= +github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= +github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40= +github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc= @@ -121,10 +155,12 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 h1:uX1JmpONuD549D73r6cgnxyUu18Zb7yHAy5AYU0Pm4Q= github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -165,6 +201,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -263,17 +303,25 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -285,9 +333,12 @@ github.com/google/certificate-transparency-go v1.3.2 h1:9ahSNZF2o7SYMaKaXhAumVEz github.com/google/certificate-transparency-go v1.3.2/go.mod h1:H5FpMUaGa5Ab2+KCYsxg6sELw3Flkl7pGZzWdBoYLXs= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -304,12 +355,13 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a h1://KbezygeMJZCSHH+HgUZiTeSoiuFspbMg1ge+eFj18= -github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= +github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY= +github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/trillian v1.7.2 h1:EPBxc4YWY4Ak8tcuhyFleY+zYlbCDCa4Sn24e1Ka8Js= github.com/google/trillian v1.7.2/go.mod h1:mfQJW4qRH6/ilABtPYNBerVJAJ/upxHLX81zxNQw05s= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= @@ -510,6 +562,18 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0= +github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -577,6 +641,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= +github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE= +github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -638,6 +706,7 @@ github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4 github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= @@ -739,6 +808,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -811,6 +881,8 @@ gitlab.com/gitlab-org/api/client-go v1.14.0 h1:0TAU8zwN4p6ZMUnXLUEkSRmUr+mN4B3JQ gitlab.com/gitlab-org/api/client-go v1.14.0/go.mod h1:adtVJ4zSTEJ2fP5Pb1zF4Ox1OKFg0MH43yxpb0T0248= go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= @@ -858,8 +930,12 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -868,13 +944,17 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= @@ -889,9 +969,11 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -900,6 +982,7 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -956,8 +1039,12 @@ golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -976,12 +1063,22 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc= google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc= google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc= google.golang.org/genproto/googleapis/api v0.0.0-20260114163908-3f89685c29c3 h1:X9z6obt+cWRX8XjDVOn+SZWhWe5kZHm46TThU9j+jss= google.golang.org/genproto/googleapis/api v0.0.0-20260114163908-3f89685c29c3/go.mod h1:dd646eSK+Dk9kxVBl1nChEOhJPtMXriCcVb4x3o6J+E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260114163908-3f89685c29c3 h1:C4WAdL+FbjnGlpp2S+HMVhBeCq2Lcib4xZqfPNF6OoQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260114163908-3f89685c29c3/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds= @@ -991,7 +1088,10 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= @@ -1019,6 +1119,8 @@ gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= diff --git a/server/runtime/docker/discoverer.go b/server/runtime/docker/discoverer.go new file mode 100644 index 000000000..a55db0c41 --- /dev/null +++ b/server/runtime/docker/discoverer.go @@ -0,0 +1,94 @@ +package docker + +import ( + "context" + "fmt" + + corev1 "github.com/agntcy/dir/api/core/v1" + runtimev1 "github.com/agntcy/dir/api/runtime/v1" + "github.com/agntcy/dir/server/types" + "github.com/agntcy/dir/utils/logging" + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/pkg/namespaces" +) + +var logger = logging.Logger("runtime") + +type discoverer struct { + store types.StoreAPI + client *containerd.Client +} + +func New(store types.StoreAPI) (types.DiscoveryAPI, error) { + client, err := containerd.New("/var/run/containerd/containerd.sock") + if err != nil { + return nil, fmt.Errorf("failed to create containerd client: %w", err) + } + + return &discoverer{ + store: store, + client: client, + }, nil +} + +func (d *discoverer) Type() string { return "containerd" } + +func (d *discoverer) IsReady(ctx context.Context) bool { + _, err := d.client.Version(ctx) + if err != nil { + return false + } + + return true +} + +func (d *discoverer) Stop() error { + if err := d.client.Close(); err != nil { + return fmt.Errorf("failed to close client: %w", err) + } + + return nil +} + +func (d *discoverer) ListProcesses(ctx context.Context, req *runtimev1.ListProcessesRequest, handlerFn func(*runtimev1.Process) error) error { + // use "moby" namespace for containers + ctx = namespaces.WithNamespace(ctx, "moby") + + // list containers + containers, err := d.client.ContainerService().List(ctx) + if err != nil { + return fmt.Errorf("failed to list containers: %w", err) + } + + // process containers + for _, container := range containers { + // // serialize to json + // data, err := json.Marshal(container) + // if err != nil { + // return fmt.Errorf("failed to marshal container data: %w", err) + // } + + // // store to labels + // labels := container.Labels + // if labels == nil { + // labels = make(map[string]string) + // } + // labels["containerd/container"] = string(data) + + // construct process + process := &runtimev1.Process{ + Pid: container.ID, + Runtime: d.Type(), + Annotations: container.Labels, + CreatedAt: container.CreatedAt.String(), + Record: &corev1.RecordMeta{}, + } + + // call handler function + if err := handlerFn(process); err != nil { + return fmt.Errorf("handler function error: %w", err) + } + } + + return nil +} diff --git a/server/runtime/docker/testdata/docker-compose.yml b/server/runtime/docker/testdata/docker-compose.yml new file mode 100644 index 000000000..3cecbc189 --- /dev/null +++ b/server/runtime/docker/testdata/docker-compose.yml @@ -0,0 +1,50 @@ +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + # Run as root to access Docker socket + user: root + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - prometheus-data:/prometheus + ports: + - '9090:9090' + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.enable-lifecycle' + - '--query.lookback-delta=15s' + restart: unless-stopped + + json-exporter: + image: prometheuscommunity/json-exporter:v0.7.0 + container_name: json-exporter + volumes: + - ./json_exporter.yml:/config.yml:ro + ports: + - '7979:7979' + command: + - '--config.file=/config.yml' + extra_hosts: + - 'host.docker.internal:host-gateway' + restart: unless-stopped + + a2a-agent-single: + image: a2a-ui + ports: + - '9999' + restart: unless-stopped + + a2a-agent-replicated: + image: a2a-ui + ports: + - '9999' + deploy: + mode: replicated + replicas: 3 + endpoint_mode: vip + restart: unless-stopped + +volumes: + prometheus-data: diff --git a/server/runtime/docker/testdata/json_exporter.yml b/server/runtime/docker/testdata/json_exporter.yml new file mode 100644 index 000000000..4c5c57341 --- /dev/null +++ b/server/runtime/docker/testdata/json_exporter.yml @@ -0,0 +1,19 @@ +modules: + a2a_agent: + headers: + Accept: application/json + metrics: + - name: a2a_agent + type: object + help: "A2A Agent information from agent card" + path: '{ $ }' + labels: + type: 'a2a' + name: '{ $.name }' + description: '{ $.description }' + version: '{ $.version }' + protocolVersion: '{ $.protocolVersion }' + preferredTransport: '{ $.preferredTransport }' + streaming: '{ $.capabilities.streaming }' + values: + up: 1 diff --git a/server/runtime/docker/testdata/prometheus.yml b/server/runtime/docker/testdata/prometheus.yml new file mode 100644 index 000000000..59e3abc03 --- /dev/null +++ b/server/runtime/docker/testdata/prometheus.yml @@ -0,0 +1,51 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + scrape_timeout: 2s + +scrape_configs: + # Probe containers with exposed ports for A2A agents + - job_name: 'a2a-agent-probe' + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + filters: + - name: status + values: ['running'] + + relabel_configs: + # Skip prometheus and blackbox containers + - source_labels: [__meta_docker_container_name] + regex: '/(prometheus|blackbox-exporter|json-exporter)' + action: drop + + # Only keep targets with a public (published) port + - source_labels: [__meta_docker_port_public] + regex: '.+' + action: keep + + # Build probe URL using container IP and public port + - source_labels: [__meta_docker_network_ip, __meta_docker_port_public] + regex: '(.+);(.+)' + replacement: 'http://host.docker.internal:${2}/.well-known/agent-card.json' + target_label: __param_target + + # Set url label to public URL + - source_labels: [__meta_docker_network_ip, __meta_docker_port_public] + regex: '(.+);(.+)' + replacement: 'http://localhost:${2}' + target_label: url + + # Route through blackbox exporter + - target_label: __address__ + replacement: json-exporter:7979 + # replacement: blackbox-exporter:9115 + + metrics_path: /probe + params: + module: [a2a_agent] + + metric_relabel_configs: + # Drop instance and job labels from stored metrics + - action: labeldrop + regex: 'instance|job' diff --git a/server/runtime/runtime.go b/server/runtime/runtime.go new file mode 100644 index 000000000..2d6aa5536 --- /dev/null +++ b/server/runtime/runtime.go @@ -0,0 +1,10 @@ +package runtime + +import ( + "github.com/agntcy/dir/server/runtime/docker" + "github.com/agntcy/dir/server/types" +) + +func New(store types.StoreAPI) (types.DiscoveryAPI, error) { + return docker.New(store) +} diff --git a/server/server.go b/server/server.go index 19a9f885a..819c45b12 100644 --- a/server/server.go +++ b/server/server.go @@ -17,6 +17,7 @@ import ( eventsv1 "github.com/agntcy/dir/api/events/v1" namingv1 "github.com/agntcy/dir/api/naming/v1" routingv1 "github.com/agntcy/dir/api/routing/v1" + runtimev1 "github.com/agntcy/dir/api/runtime/v1" searchv1 "github.com/agntcy/dir/api/search/v1" signv1 "github.com/agntcy/dir/api/sign/v1" storev1 "github.com/agntcy/dir/api/store/v1" @@ -37,6 +38,7 @@ import ( "github.com/agntcy/dir/server/publication" "github.com/agntcy/dir/server/reverification" "github.com/agntcy/dir/server/routing" + "github.com/agntcy/dir/server/runtime" "github.com/agntcy/dir/server/signing" "github.com/agntcy/dir/server/store" "github.com/agntcy/dir/server/sync" @@ -68,6 +70,7 @@ type Server struct { authzService *authz.Service publicationService *publication.Service reverificationService *reverification.Service + runtimeService types.DiscoveryAPI health *healthcheck.Checker grpcServer *grpc.Server metricsServer *metrics.Server @@ -286,6 +289,12 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) { return nil, fmt.Errorf("failed to create signing service: %w", err) } + // Create runtime service + runtimeService, err := runtime.New(storeAPI) + if err != nil { + return nil, fmt.Errorf("failed to create runtime service: %w", err) + } + // Create a server grpcServer := grpc.NewServer(serverOpts...) @@ -321,6 +330,7 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) { namingProvider, controller.WithVerificationTTL(options.Config().Reverification.GetTTL()), )) + runtimev1.RegisterDiscoveryServiceServer(grpcServer, controller.NewRuntimeController(runtimeService)) // Register health service healthChecker.Register(grpcServer) @@ -349,6 +359,7 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) { health: healthChecker, grpcServer: grpcServer, metricsServer: metricsServer, + runtimeService: runtimeService, }, nil } @@ -382,6 +393,13 @@ func (s Server) Close(ctx context.Context) { } } + // Stop runtime service + if s.runtimeService != nil { + if err := s.runtimeService.Stop(); err != nil { + logger.Error("Failed to stop runtime service", "error", err) + } + } + // Stop metrics server if s.metricsServer != nil { stopCtx, cancel := context.WithTimeout(ctx, 10*time.Second) //nolint:mnd @@ -486,6 +504,7 @@ func (s Server) start(ctx context.Context) error { s.health.AddReadinessCheck("publication", s.publicationService.IsReady) s.health.AddReadinessCheck("store", s.store.IsReady) s.health.AddReadinessCheck("routing", s.routing.IsReady) + s.health.AddReadinessCheck("runtime", s.runtimeService.IsReady) // Start health check monitoring if err := s.health.Start(ctx); err != nil { diff --git a/server/types/runtime.go b/server/types/runtime.go new file mode 100644 index 000000000..6254e5d91 --- /dev/null +++ b/server/types/runtime.go @@ -0,0 +1,19 @@ +package types + +import ( + "context" + + runtimev1 "github.com/agntcy/dir/api/runtime/v1" +) + +type DiscoveryAPI interface { + ListProcesses( + ctx context.Context, + req *runtimev1.ListProcessesRequest, + fn func(*runtimev1.Process) error, + ) error + + IsReady(context.Context) bool + + Stop() error +}