From 7a9d7bcdc381d55da47e9f4ae16a5aef519a5e07 Mon Sep 17 00:00:00 2001 From: Kathryn Baldauf Date: Thu, 28 Oct 2021 20:18:03 -0700 Subject: [PATCH 1/3] Update ncproxy to include new ncproxy network and endpoint types Signed-off-by: Kathryn Baldauf --- computeagent/computeagent.pb.go | 128 +++++++---- computeagent/computeagent.proto | 1 + devices/assigned_devices.go | 2 +- devices/drivers.go | 21 +- devices/pnp.go | 27 +++ hcsoci/devices.go | 2 +- ncproxynetworking/endpoints.go | 33 +++ ncproxynetworking/networks.go | 17 ++ ncproxystore/buckets.go | 77 +++++++ ncproxystore/store.go | 279 ++++++++++++++++++++++++ ncproxystore/store_test.go | 284 ++++++++++++++++++++++++ tools/networkagent/defs.go | 10 +- tools/networkagent/main.go | 372 +++++++++++++++++++++----------- uvm/computeagent.go | 31 ++- uvm/computeagent_test.go | 11 +- uvm/network.go | 32 +++ uvm/virtual_device.go | 15 +- 17 files changed, 1159 insertions(+), 183 deletions(-) create mode 100644 ncproxynetworking/endpoints.go create mode 100644 ncproxynetworking/networks.go create mode 100644 ncproxystore/buckets.go create mode 100644 ncproxystore/store.go create mode 100644 ncproxystore/store_test.go diff --git a/computeagent/computeagent.pb.go b/computeagent/computeagent.pb.go index b3dbd34510..8d7236dffd 100644 --- a/computeagent/computeagent.pb.go +++ b/computeagent/computeagent.pb.go @@ -30,6 +30,7 @@ type AssignPCIInternalRequest struct { ContainerID string `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` DeviceID string `protobuf:"bytes,2,opt,name=device_id,json=deviceId,proto3" json:"device_id,omitempty"` VirtualFunctionIndex uint32 `protobuf:"varint,3,opt,name=virtual_function_index,json=virtualFunctionIndex,proto3" json:"virtual_function_index,omitempty"` + NicID string `protobuf:"bytes,4,opt,name=nic_id,json=nicId,proto3" json:"nic_id,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -482,48 +483,48 @@ func init() { } var fileDescriptor_7f2f03dc308add4c = []byte{ - // 642 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x95, 0x4d, 0x6f, 0xd3, 0x30, - 0x1c, 0xc6, 0xeb, 0xa2, 0x55, 0xab, 0xdb, 0x09, 0xf0, 0x46, 0x69, 0x32, 0xa9, 0x9d, 0x7a, 0x1a, - 0x1c, 0x9c, 0x69, 0xe3, 0x36, 0x24, 0xd4, 0x35, 0x20, 0xf9, 0xb0, 0x31, 0xc2, 0x81, 0x97, 0x4b, - 0x95, 0xc5, 0x6e, 0x6a, 0xa9, 0xb5, 0x43, 0xe2, 0x44, 0xf4, 0xc6, 0x17, 0x41, 0x48, 0x9c, 0x91, - 0xf8, 0x1a, 0x3b, 0x72, 0xe4, 0x54, 0xb1, 0xf0, 0x45, 0x50, 0x5e, 0x9a, 0x0d, 0x48, 0xa4, 0x21, - 0x71, 0x80, 0x5b, 0xfc, 0xff, 0xdb, 0xee, 0xef, 0xb1, 0x1f, 0x3f, 0x85, 0x8f, 0x5d, 0xae, 0xa6, - 0xe1, 0x19, 0x76, 0xe4, 0xdc, 0x38, 0xe6, 0x8e, 0x2f, 0x03, 0x39, 0x51, 0xc6, 0xd4, 0x09, 0x82, - 0x29, 0x9f, 0x1b, 0x5c, 0x28, 0xe6, 0x0b, 0x7b, 0x66, 0x38, 0x72, 0xee, 0x85, 0x8a, 0xd9, 0x2e, - 0x13, 0xea, 0xa7, 0x01, 0xf6, 0x7c, 0xa9, 0xa4, 0xbe, 0xe5, 0x4a, 0x57, 0xa6, 0x9f, 0x46, 0xf2, - 0x95, 0x57, 0x35, 0x57, 0x4a, 0x77, 0xc6, 0x8c, 0x74, 0x74, 0x16, 0x4e, 0x0c, 0x5b, 0x2c, 0xb2, - 0xd6, 0xe0, 0x33, 0x80, 0xdd, 0x61, 0x10, 0x70, 0x57, 0x9c, 0x8e, 0x08, 0xc9, 0x7f, 0xc6, 0x62, - 0x6f, 0x42, 0x16, 0x28, 0xb4, 0x0f, 0xdb, 0x8e, 0x14, 0xca, 0xe6, 0x82, 0xf9, 0x63, 0x4e, 0xbb, - 0x60, 0x07, 0xec, 0x36, 0x8f, 0x6e, 0xc6, 0xcb, 0x7e, 0x6b, 0xb4, 0xaa, 0x13, 0xd3, 0x6a, 0x15, - 0x93, 0x08, 0x45, 0xf7, 0x60, 0x93, 0xb2, 0x88, 0x3b, 0x2c, 0x59, 0x50, 0x4f, 0x17, 0xb4, 0xe3, - 0x65, 0x7f, 0xdd, 0x4c, 0x8b, 0xc4, 0xb4, 0xd6, 0xb3, 0x36, 0xa1, 0xe8, 0x01, 0xec, 0x44, 0xdc, - 0x57, 0xa1, 0x3d, 0x1b, 0x4f, 0x42, 0xe1, 0x28, 0x2e, 0xc5, 0x98, 0x0b, 0xca, 0xde, 0x76, 0x6f, - 0xec, 0x80, 0xdd, 0x0d, 0x6b, 0x2b, 0xef, 0x3e, 0xc9, 0x9b, 0x24, 0xe9, 0x0d, 0x0e, 0xa0, 0x56, - 0x02, 0x1c, 0x78, 0x52, 0x04, 0x0c, 0x75, 0x60, 0xbd, 0xe0, 0x6c, 0xc4, 0xcb, 0x7e, 0x9d, 0x98, - 0x56, 0x9d, 0xd3, 0x54, 0xa6, 0xc5, 0xe6, 0x32, 0x62, 0xff, 0x8b, 0xcc, 0x6d, 0xa8, 0x95, 0x00, - 0x67, 0x32, 0x07, 0x1f, 0x00, 0xbc, 0x33, 0xa4, 0xf4, 0x84, 0x8c, 0xfe, 0x86, 0x96, 0x1d, 0xd8, - 0x10, 0xdc, 0xb9, 0x14, 0xd2, 0x8c, 0x97, 0xfd, 0xb5, 0x13, 0xee, 0x10, 0xd3, 0x5a, 0x13, 0xdc, - 0x21, 0x14, 0xed, 0xc1, 0x75, 0x26, 0xa8, 0x27, 0xb9, 0x50, 0x29, 0x74, 0x6b, 0x7f, 0x0b, 0x67, - 0x9e, 0xc2, 0x2b, 0x4f, 0xe1, 0xa1, 0x58, 0x58, 0xc5, 0xac, 0x41, 0x17, 0x76, 0x7e, 0x05, 0xcc, - 0xd9, 0x3f, 0x01, 0xd8, 0x3d, 0x96, 0x94, 0x4f, 0x16, 0x25, 0xf8, 0x97, 0x28, 0xe0, 0x1a, 0x28, - 0xf5, 0xeb, 0xa0, 0xa0, 0x87, 0x70, 0x93, 0xcb, 0x68, 0xec, 0xc9, 0x19, 0x77, 0x16, 0xe3, 0x80, - 0x29, 0xc5, 0x85, 0x1b, 0xe4, 0x3a, 0xda, 0x98, 0xc8, 0xe8, 0x79, 0x5e, 0xb3, 0x6e, 0x73, 0x19, - 0x9d, 0xa6, 0xf3, 0x56, 0xa5, 0xe4, 0x1e, 0x4a, 0x68, 0x73, 0x2d, 0x1f, 0x01, 0xec, 0x9a, 0x6c, - 0xc6, 0x14, 0xfb, 0x87, 0xaf, 0x62, 0x1b, 0x6a, 0x25, 0x8c, 0xb9, 0x82, 0xf7, 0x00, 0xb6, 0xae, - 0x9c, 0x00, 0xba, 0x0f, 0x6f, 0x11, 0x19, 0x3d, 0x9d, 0x4c, 0x66, 0xd2, 0xa6, 0x2f, 0x18, 0x77, - 0xa7, 0x2a, 0x05, 0xdf, 0xb0, 0x7e, 0xab, 0xa3, 0x3d, 0xb8, 0xf9, 0x2c, 0x64, 0x21, 0x3b, 0xb5, - 0xb9, 0x1f, 0xe4, 0xaa, 0x59, 0x46, 0xbe, 0x61, 0x95, 0xb5, 0x92, 0x15, 0x29, 0x81, 0x1f, 0x7a, - 0xea, 0x58, 0x52, 0xe6, 0xdb, 0x89, 0xe1, 0xf3, 0x77, 0x50, 0xd6, 0xda, 0xff, 0x5e, 0x87, 0xed, - 0x51, 0x96, 0x73, 0xc3, 0x24, 0xe7, 0xd0, 0x21, 0x6c, 0x64, 0xc6, 0x42, 0x1d, 0x5c, 0xfa, 0x04, - 0xf4, 0xbb, 0xb8, 0xc2, 0x79, 0x35, 0x64, 0xc2, 0x66, 0x71, 0x99, 0x48, 0xc3, 0x55, 0x36, 0xd4, - 0x75, 0x5c, 0x7d, 0xe7, 0xe9, 0x2e, 0xc5, 0x81, 0x22, 0x0d, 0x57, 0x19, 0x40, 0xd7, 0x71, 0xf5, - 0xb9, 0xa7, 0xbb, 0x14, 0x39, 0x86, 0x34, 0x5c, 0x15, 0xc2, 0xba, 0x8e, 0x2b, 0xe3, 0x2e, 0xdb, - 0xa5, 0x88, 0x09, 0xa4, 0xe1, 0xaa, 0x8c, 0xd3, 0x75, 0x5c, 0x9d, 0x26, 0xb5, 0xa3, 0x57, 0xe7, - 0x17, 0xbd, 0xda, 0xd7, 0x8b, 0x5e, 0xed, 0x5d, 0xdc, 0x03, 0xe7, 0x71, 0x0f, 0x7c, 0x89, 0x7b, - 0xe0, 0x5b, 0xdc, 0x03, 0xaf, 0x1f, 0xfd, 0xf9, 0xff, 0xd2, 0xe1, 0xd5, 0xc1, 0xcb, 0xda, 0x59, - 0x23, 0xf5, 0xe5, 0xc1, 0x8f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x92, 0xce, 0xd7, 0x96, 0xe3, 0x06, - 0x00, 0x00, + // 655 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x55, 0xcf, 0x6f, 0xd3, 0x30, + 0x14, 0xae, 0x0b, 0xab, 0x56, 0xb7, 0x13, 0xe0, 0x8d, 0x92, 0x64, 0x52, 0x3b, 0xf5, 0x34, 0x38, + 0x38, 0xd3, 0xc6, 0x6d, 0x48, 0xa8, 0x6b, 0x40, 0xf2, 0x61, 0x63, 0x84, 0x03, 0x3f, 0x2e, 0x55, + 0x16, 0xbb, 0xa9, 0xa5, 0xd6, 0x2e, 0x89, 0x13, 0xd1, 0x1b, 0xff, 0x08, 0x42, 0xe2, 0x8c, 0xc4, + 0xbf, 0xb1, 0x23, 0xdc, 0x38, 0x55, 0x2c, 0xfc, 0x23, 0x28, 0x3f, 0x96, 0x0d, 0x48, 0xa4, 0x21, + 0x71, 0x80, 0x9b, 0xfd, 0x9e, 0xfd, 0xf2, 0x7d, 0xef, 0x7d, 0xfe, 0x02, 0x1f, 0x79, 0x5c, 0x4d, + 0xc2, 0x13, 0xec, 0xca, 0x99, 0x79, 0xc8, 0x5d, 0x5f, 0x06, 0x72, 0xac, 0xcc, 0x89, 0x1b, 0x04, + 0x13, 0x3e, 0x33, 0xb9, 0x50, 0xcc, 0x17, 0xce, 0xd4, 0x74, 0xe5, 0x6c, 0x1e, 0x2a, 0xe6, 0x78, + 0x4c, 0xa8, 0x9f, 0x36, 0x78, 0xee, 0x4b, 0x25, 0x8d, 0x0d, 0x4f, 0x7a, 0x32, 0x5d, 0x9a, 0xc9, + 0x2a, 0x8f, 0xea, 0x9e, 0x94, 0xde, 0x94, 0x99, 0xe9, 0xee, 0x24, 0x1c, 0x9b, 0x8e, 0x58, 0x64, + 0xa9, 0xfe, 0x17, 0x00, 0xb5, 0x41, 0x10, 0x70, 0x4f, 0x1c, 0x0f, 0x09, 0xc9, 0x3f, 0x63, 0xb3, + 0xd7, 0x21, 0x0b, 0x14, 0xda, 0x85, 0x6d, 0x57, 0x0a, 0xe5, 0x70, 0xc1, 0xfc, 0x11, 0xa7, 0x1a, + 0xd8, 0x02, 0xdb, 0xcd, 0x83, 0x1b, 0xf1, 0xb2, 0xd7, 0x1a, 0x9e, 0xc7, 0x89, 0x65, 0xb7, 0x8a, + 0x43, 0x84, 0xa2, 0xbb, 0xb0, 0x49, 0x59, 0xc4, 0x5d, 0x96, 0x5c, 0xa8, 0xa7, 0x17, 0xda, 0xf1, + 0xb2, 0xb7, 0x6a, 0xa5, 0x41, 0x62, 0xd9, 0xab, 0x59, 0x9a, 0x50, 0x74, 0x1f, 0x76, 0x22, 0xee, + 0xab, 0xd0, 0x99, 0x8e, 0xc6, 0xa1, 0x70, 0x15, 0x97, 0x62, 0xc4, 0x05, 0x65, 0x6f, 0xb4, 0x6b, + 0x5b, 0x60, 0x7b, 0xcd, 0xde, 0xc8, 0xb3, 0x8f, 0xf3, 0x24, 0x49, 0x72, 0x68, 0x0b, 0x36, 0x04, + 0x77, 0x93, 0xea, 0xd7, 0xd3, 0xea, 0xcd, 0x78, 0xd9, 0x5b, 0x39, 0xe2, 0x2e, 0xb1, 0xec, 0x15, + 0xc1, 0x5d, 0x42, 0xfb, 0x7b, 0x50, 0x2f, 0xa1, 0x14, 0xcc, 0xa5, 0x08, 0x18, 0xea, 0xc0, 0x7a, + 0xc1, 0xa4, 0x11, 0x2f, 0x7b, 0x75, 0x62, 0xd9, 0x75, 0x4e, 0xfb, 0x9f, 0x00, 0xd4, 0x6c, 0x36, + 0x93, 0x11, 0xfb, 0x4f, 0x1a, 0xd1, 0xdf, 0x84, 0x7a, 0x09, 0xe0, 0x8c, 0x66, 0xff, 0x3d, 0x80, + 0xb7, 0x07, 0x94, 0x1e, 0x91, 0xe1, 0xdf, 0xe0, 0x72, 0xd1, 0xf3, 0x7a, 0x79, 0xcf, 0xd1, 0x0e, + 0x5c, 0x65, 0x82, 0xce, 0x25, 0x17, 0x2a, 0x05, 0xdd, 0xda, 0xdd, 0xc0, 0x99, 0xea, 0xf0, 0xb9, + 0xea, 0xf0, 0x40, 0x2c, 0xec, 0xe2, 0x54, 0x5f, 0x83, 0x9d, 0x5f, 0x01, 0xe6, 0xd8, 0x3f, 0x02, + 0xa8, 0x1d, 0x4a, 0xca, 0xc7, 0x8b, 0x12, 0xf8, 0x17, 0x50, 0xc0, 0x15, 0xa0, 0xd4, 0xaf, 0x02, + 0x05, 0x3d, 0x80, 0xeb, 0x5c, 0x46, 0xa3, 0xb9, 0x9c, 0x72, 0x77, 0x31, 0x0a, 0x98, 0x52, 0x5c, + 0x78, 0x41, 0xce, 0xa3, 0x8d, 0x89, 0x8c, 0x9e, 0xe5, 0x31, 0xfb, 0x16, 0x97, 0xd1, 0x71, 0x7a, + 0xee, 0x3c, 0x94, 0xcc, 0xa1, 0x04, 0x6d, 0xce, 0xe5, 0x03, 0x80, 0x9a, 0xc5, 0xa6, 0x4c, 0xb1, + 0x7f, 0x78, 0x14, 0x9b, 0x50, 0x2f, 0xc1, 0x98, 0x33, 0x78, 0x07, 0x60, 0xeb, 0x52, 0x07, 0xd0, + 0x3d, 0x78, 0x93, 0xc8, 0xe8, 0xc9, 0x78, 0x3c, 0x95, 0x0e, 0x7d, 0xce, 0xb8, 0x37, 0x51, 0x29, + 0xf0, 0x35, 0xfb, 0xb7, 0x38, 0xda, 0x81, 0xeb, 0x4f, 0x43, 0x16, 0xb2, 0x63, 0x87, 0xfb, 0x41, + 0xce, 0x9a, 0x65, 0xc8, 0xd7, 0xec, 0xb2, 0x54, 0x72, 0x23, 0x45, 0xe0, 0x87, 0x73, 0x75, 0x28, + 0x29, 0xf3, 0x9d, 0x44, 0xf0, 0xf9, 0x3b, 0x28, 0x4b, 0xed, 0x7e, 0xaf, 0xc3, 0xf6, 0x30, 0x73, + 0xc2, 0x41, 0xe2, 0x84, 0x68, 0x1f, 0x36, 0x32, 0x61, 0xa1, 0x0e, 0x2e, 0x7d, 0x02, 0xc6, 0x1d, + 0x5c, 0xa1, 0xbc, 0x1a, 0xb2, 0x60, 0xb3, 0x18, 0x26, 0xd2, 0x71, 0x95, 0x0c, 0x0d, 0x03, 0x57, + 0xcf, 0x3c, 0xad, 0x52, 0x34, 0x14, 0xe9, 0xb8, 0x4a, 0x00, 0x86, 0x81, 0xab, 0xfb, 0x9e, 0x56, + 0x29, 0x7c, 0x0c, 0xe9, 0xb8, 0xca, 0xa6, 0x0d, 0x03, 0x57, 0xda, 0x5d, 0x56, 0xa5, 0xb0, 0x09, + 0xa4, 0xe3, 0x2a, 0x8f, 0x33, 0x0c, 0x5c, 0xed, 0x26, 0xb5, 0x83, 0x97, 0xa7, 0x67, 0xdd, 0xda, + 0xd7, 0xb3, 0x6e, 0xed, 0x6d, 0xdc, 0x05, 0xa7, 0x71, 0x17, 0x7c, 0x8e, 0xbb, 0xe0, 0x5b, 0xdc, + 0x05, 0xaf, 0x1e, 0xfe, 0xf9, 0x9f, 0x6b, 0xff, 0xf2, 0xe6, 0x45, 0xed, 0xa4, 0x91, 0xea, 0x72, + 0xef, 0x47, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe4, 0x25, 0x88, 0x14, 0x05, 0x07, 0x00, 0x00, } func (m *AssignPCIInternalRequest) Marshal() (dAtA []byte, err error) { @@ -558,6 +559,12 @@ func (m *AssignPCIInternalRequest) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintComputeagent(dAtA, i, uint64(m.VirtualFunctionIndex)) } + if len(m.NicID) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintComputeagent(dAtA, i, uint64(len(m.NicID))) + i += copy(dAtA[i:], m.NicID) + } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -908,6 +915,10 @@ func (m *AssignPCIInternalRequest) Size() (n int) { if m.VirtualFunctionIndex != 0 { n += 1 + sovComputeagent(uint64(m.VirtualFunctionIndex)) } + l = len(m.NicID) + if l > 0 { + n += 1 + l + sovComputeagent(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -1115,6 +1126,7 @@ func (this *AssignPCIInternalRequest) String() string { `ContainerID:` + fmt.Sprintf("%v", this.ContainerID) + `,`, `DeviceID:` + fmt.Sprintf("%v", this.DeviceID) + `,`, `VirtualFunctionIndex:` + fmt.Sprintf("%v", this.VirtualFunctionIndex) + `,`, + `NicID:` + fmt.Sprintf("%v", this.NicID) + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -1454,6 +1466,38 @@ func (m *AssignPCIInternalRequest) Unmarshal(dAtA []byte) error { break } } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NicID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowComputeagent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthComputeagent + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthComputeagent + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NicID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipComputeagent(dAtA[iNdEx:]) diff --git a/computeagent/computeagent.proto b/computeagent/computeagent.proto index 25d5bb84ee..51127ffd01 100644 --- a/computeagent/computeagent.proto +++ b/computeagent/computeagent.proto @@ -17,6 +17,7 @@ message AssignPCIInternalRequest { string container_id = 1; string device_id = 2; uint32 virtual_function_index = 3; + string nic_id = 4; } message AssignPCIInternalResponse { diff --git a/devices/assigned_devices.go b/devices/assigned_devices.go index 3f4f3c27b2..dd0b2ee66f 100644 --- a/devices/assigned_devices.go +++ b/devices/assigned_devices.go @@ -38,7 +38,7 @@ func AddDevice(ctx context.Context, vm *uvm.UtilityVM, idType, deviceID string, } }() if idType == uvm.VPCIDeviceIDType || idType == uvm.VPCIDeviceIDTypeLegacy { - vpci, err = vm.AssignDevice(ctx, deviceID, index) + vpci, err = vm.AssignDevice(ctx, deviceID, index, "") if err != nil { return vpci, nil, errors.Wrapf(err, "failed to assign device %s of type %s to pod %s", deviceID, idType, vm.ID()) } diff --git a/devices/drivers.go b/devices/drivers.go index f37911efe6..2a2d01af84 100644 --- a/devices/drivers.go +++ b/devices/drivers.go @@ -59,10 +59,10 @@ func execModprobeInstallDriver(ctx context.Context, vm *uvm.UtilityVM, driverDir } defer l.Close() - var pipeResults []string + var stderrOutput string errChan := make(chan error) - go readCsPipeOutput(l, errChan, &pipeResults) + go readAllPipeOutput(l, errChan, &stderrOutput) args := []string{ "/bin/install-drivers", @@ -73,19 +73,24 @@ func execModprobeInstallDriver(ctx context.Context, vm *uvm.UtilityVM, driverDir Stderr: p, } - exitCode, err := cmd.ExecInUvm(ctx, vm, req) - if err != nil && err != noExecOutputErr { - return errors.Wrapf(err, "failed to install driver %s in uvm with exit code %d", driverDir, exitCode) - } + var ( + execErr error + exitCode int + ) + exitCode, execErr = cmd.ExecInUvm(ctx, vm, req) // wait to finish parsing stdout results select { case err := <-errChan: if err != nil { - return err + return errors.Wrap(err, execErr.Error()) } case <-ctx.Done(): - return ctx.Err() + return errors.Wrap(ctx.Err(), execErr.Error()) + } + + if execErr != nil && execErr != noExecOutputErr { + return errors.Wrapf(execErr, "failed to install driver %s in uvm with exit code %d: %v", driverDir, exitCode, stderrOutput) } log.G(ctx).WithField("added drivers", driverDir).Debug("installed drivers") diff --git a/devices/pnp.go b/devices/pnp.go index 4714090839..ce30bf7865 100644 --- a/devices/pnp.go +++ b/devices/pnp.go @@ -95,3 +95,30 @@ func readCsPipeOutput(l net.Listener, errChan chan<- error, result *[]string) { errChan <- nil } + +// readCsPipeOutput is a helper function that connects to a listener and reads +// the connection's comma separated output until done. resulting comma separated +// values are returned in the `result` param. The `errChan` param is used to +// propagate an errors to the calling function.f +func readAllPipeOutput(l net.Listener, errChan chan<- error, result *string) { + defer close(errChan) + c, err := l.Accept() + if err != nil { + errChan <- errors.Wrapf(err, "failed to accept named pipe") + return + } + bytes, err := ioutil.ReadAll(c) + if err != nil { + errChan <- err + return + } + + *result = string(bytes) + + if len(*result) == 0 { + errChan <- noExecOutputErr + return + } + + errChan <- nil +} diff --git a/hcsoci/devices.go b/hcsoci/devices.go index 202435af7c..42d11aac52 100644 --- a/hcsoci/devices.go +++ b/hcsoci/devices.go @@ -195,7 +195,7 @@ func handleAssignedDevicesLCOW( case uvm.VPCIDeviceIDType, uvm.VPCIDeviceIDTypeLegacy, uvm.GPUDeviceIDType: gpuPresent = gpuPresent || d.IDType == uvm.GPUDeviceIDType pciID, index := getDeviceInfoFromPath(d.ID) - vpci, err := vm.AssignDevice(ctx, pciID, index) + vpci, err := vm.AssignDevice(ctx, pciID, index, "") if err != nil { return resultDevs, closers, errors.Wrapf(err, "failed to assign device %s, function %d to pod %s", pciID, index, vm.ID()) } diff --git a/ncproxynetworking/endpoints.go b/ncproxynetworking/endpoints.go new file mode 100644 index 0000000000..e29e04824b --- /dev/null +++ b/ncproxynetworking/endpoints.go @@ -0,0 +1,33 @@ +package ncproxynetworking + +type Endpoint struct { + EndpointName string + NamespaceID string + Settings *EndpointSettings +} + +type EndpointSettings struct { + Name string + Macaddress string + IPAddress string + IPAddressPrefixLength uint32 + NetworkName string + DefaultGateway string + DeviceDetails *DeviceDetails +} + +type DeviceDetails struct { + PCIDeviceDetails *PCIDeviceDetails +} + +type PCIDeviceDetails struct { + DeviceID string + VirtualFunctionIndex uint32 +} + +func NewEndpoint(settings *EndpointSettings) (*Endpoint, error) { + return &Endpoint{ + EndpointName: settings.Name, + Settings: settings, + }, nil +} diff --git a/ncproxynetworking/networks.go b/ncproxynetworking/networks.go new file mode 100644 index 0000000000..e494c38029 --- /dev/null +++ b/ncproxynetworking/networks.go @@ -0,0 +1,17 @@ +package ncproxynetworking + +type Network struct { + NetworkName string + Settings *NetworkSettings +} + +type NetworkSettings struct { + Name string +} + +func NewNetwork(settings *NetworkSettings) (*Network, error) { + return &Network{ + NetworkName: settings.Name, + Settings: settings, + }, nil +} diff --git a/ncproxystore/buckets.go b/ncproxystore/buckets.go new file mode 100644 index 0000000000..69e5ffbcf1 --- /dev/null +++ b/ncproxystore/buckets.go @@ -0,0 +1,77 @@ +package ncproxystore + +import ( + bolt "go.etcd.io/bbolt" +) + +const schemaVersion = "v1" + +var ( + bucketKeyVersion = []byte(schemaVersion) + + bucketKeyNetwork = []byte("network") + bucketKeyEndpoint = []byte("endpoint") + bucketKeyComputeAgent = []byte("computeagent") +) + +// Below is the current database schema. This should be updated any time the schema is +// changed or updated. The version should be incremented if breaking changes are made. +// └──v1 - Schema version bucket +// └──computeagent - Compute agent bucket +// └──containerID : - Entry in compute agent bucket: Address to +// the compute agent for containerID + +// taken from containerd/containerd/metadata/buckets.go +func getBucket(tx *bolt.Tx, keys ...[]byte) *bolt.Bucket { + bkt := tx.Bucket(keys[0]) + + for _, key := range keys[1:] { + if bkt == nil { + break + } + bkt = bkt.Bucket(key) + } + + return bkt +} + +// taken from containerd/containerd/metadata/buckets.go +func createBucketIfNotExists(tx *bolt.Tx, keys ...[]byte) (*bolt.Bucket, error) { + bkt, err := tx.CreateBucketIfNotExists(keys[0]) + if err != nil { + return nil, err + } + + for _, key := range keys[1:] { + bkt, err = bkt.CreateBucketIfNotExists(key) + if err != nil { + return nil, err + } + } + + return bkt, nil +} + +func createNetworkBucket(tx *bolt.Tx) (*bolt.Bucket, error) { + return createBucketIfNotExists(tx, bucketKeyVersion, bucketKeyNetwork) +} + +func getNetworkBucket(tx *bolt.Tx) *bolt.Bucket { + return getBucket(tx, bucketKeyVersion, bucketKeyNetwork) +} + +func createEndpointBucket(tx *bolt.Tx) (*bolt.Bucket, error) { + return createBucketIfNotExists(tx, bucketKeyVersion, bucketKeyEndpoint) +} + +func getEndpointBucket(tx *bolt.Tx) *bolt.Bucket { + return getBucket(tx, bucketKeyVersion, bucketKeyEndpoint) +} + +func createComputeAgentBucket(tx *bolt.Tx) (*bolt.Bucket, error) { + return createBucketIfNotExists(tx, bucketKeyVersion, bucketKeyComputeAgent) +} + +func getComputeAgentBucket(tx *bolt.Tx) *bolt.Bucket { + return getBucket(tx, bucketKeyVersion, bucketKeyComputeAgent) +} diff --git a/ncproxystore/store.go b/ncproxystore/store.go new file mode 100644 index 0000000000..49a4d43dce --- /dev/null +++ b/ncproxystore/store.go @@ -0,0 +1,279 @@ +package ncproxystore + +import ( + "context" + "encoding/json" + + "github.com/Microsoft/hcsshim/internal/ncproxynetworking" + "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" +) + +var ( + ErrBucketNotFound = errors.New("bucket not found") + errKeyNotFound = errors.New("key does not exist") +) + +type NetworkingStore struct { + db *bolt.DB +} + +func NewNetworkingStore(database *bolt.DB) *NetworkingStore { + return &NetworkingStore{ + db: database, + } +} + +func (n *NetworkingStore) Close() error { + return n.db.Close() +} + +func (n *NetworkingStore) GetNetworkByName(ctx context.Context, networkName string) (*ncproxynetworking.Network, error) { + internalData := &ncproxynetworking.Network{} + if err := n.db.View(func(tx *bolt.Tx) error { + bkt := getNetworkBucket(tx) + if bkt == nil { + return errors.Wrapf(ErrBucketNotFound, "network bucket %v", bucketKeyNetwork) + } + data := bkt.Get([]byte(networkName)) + if data == nil { + return errors.Wrapf(errKeyNotFound, "network %v", networkName) + } + if err := json.Unmarshal(data, internalData); err != nil { + return errors.Wrapf(err, "data is %v", string(data)) + } + return nil + }); err != nil { + return nil, err + } + return internalData, nil +} + +func (n *NetworkingStore) CreateNetwork(ctx context.Context, network *ncproxynetworking.Network) error { + if err := n.db.Update(func(tx *bolt.Tx) error { + bkt, err := createNetworkBucket(tx) + if err != nil { + return err + } + internalData, err := json.Marshal(network) + if err != nil { + return err + } + return bkt.Put([]byte(network.NetworkName), internalData) + }); err != nil { + return err + } + return nil +} + +func (n *NetworkingStore) DeleteNetwork(ctx context.Context, networkName string) error { + if err := n.db.Update(func(tx *bolt.Tx) error { + bkt := getNetworkBucket(tx) + if bkt == nil { + return errors.Wrapf(ErrBucketNotFound, "bucket %v", bucketKeyNetwork) + } + return bkt.Delete([]byte(networkName)) + }); err != nil { + return err + } + return nil +} + +func (n *NetworkingStore) ListNetworks(ctx context.Context) (results []*ncproxynetworking.Network, err error) { + if err := n.db.View(func(tx *bolt.Tx) error { + bkt := getNetworkBucket(tx) + if bkt == nil { + return errors.Wrapf(ErrBucketNotFound, "network bucket %v", bucketKeyNetwork) + } + err := bkt.ForEach(func(k, v []byte) error { + data := bkt.Get([]byte(k)) + if data == nil { + return errors.Wrapf(errKeyNotFound, "network %v", k) + } + internalData := &ncproxynetworking.Network{} + if err := json.Unmarshal(data, internalData); err != nil { + return errors.Wrapf(err, "data is %v", string(data)) + } + results = append(results, internalData) + return nil + }) + return err + }); err != nil { + return nil, err + } + + return results, nil +} + +func (n *NetworkingStore) GetEndpointByName(ctx context.Context, endpointName string) (*ncproxynetworking.Endpoint, error) { + endpt := &ncproxynetworking.Endpoint{} + if err := n.db.View(func(tx *bolt.Tx) error { + bkt := getEndpointBucket(tx) + if bkt == nil { + return errors.Wrapf(ErrBucketNotFound, "endpoint bucket %v", bucketKeyEndpoint) + } + jsonData := bkt.Get([]byte(endpointName)) + if jsonData == nil { + return errors.Wrapf(errKeyNotFound, "endpoint %v", endpointName) + } + if err := json.Unmarshal(jsonData, endpt); err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + + return endpt, nil +} + +func (n *NetworkingStore) CreatEndpoint(ctx context.Context, endpt *ncproxynetworking.Endpoint) error { + return n.updateEndpoint(ctx, endpt) +} + +func (n *NetworkingStore) UpdateEndpoint(ctx context.Context, endpt *ncproxynetworking.Endpoint) error { + return n.updateEndpoint(ctx, endpt) +} + +func (n *NetworkingStore) updateEndpoint(ctx context.Context, endpt *ncproxynetworking.Endpoint) error { + if err := n.db.Update(func(tx *bolt.Tx) error { + bkt, err := createEndpointBucket(tx) + if err != nil { + return err + } + jsonEndptData, err := json.Marshal(endpt) + if err != nil { + return err + } + return bkt.Put([]byte(endpt.EndpointName), jsonEndptData) + }); err != nil { + return err + } + return nil +} + +func (n *NetworkingStore) DeleteEndpoint(ctx context.Context, endpointName string) error { + if err := n.db.Update(func(tx *bolt.Tx) error { + bkt := getEndpointBucket(tx) + if bkt == nil { + return errors.Wrapf(ErrBucketNotFound, "bucket %v", bucketKeyEndpoint) + } + return bkt.Delete([]byte(endpointName)) + }); err != nil { + return err + } + return nil +} + +func (n *NetworkingStore) ListEndpoints(ctx context.Context) (results []*ncproxynetworking.Endpoint, err error) { + if err := n.db.View(func(tx *bolt.Tx) error { + bkt := getEndpointBucket(tx) + if bkt == nil { + return errors.Wrapf(ErrBucketNotFound, "endpoint bucket %v", bucketKeyEndpoint) + } + err := bkt.ForEach(func(k, v []byte) error { + jsonData := bkt.Get([]byte(k)) + if jsonData == nil { + return errors.Wrapf(errKeyNotFound, "endpoint %v", k) + } + endptInternal := &ncproxynetworking.Endpoint{} + if err := json.Unmarshal(jsonData, endptInternal); err != nil { + return err + } + results = append(results, endptInternal) + return nil + }) + return err + }); err != nil { + return nil, err + } + + return results, nil +} + +// ComputeAgentStore is a database that stores a key value pair of container id +// to compute agent server address +type ComputeAgentStore struct { + DB *bolt.DB +} + +func NewComputeAgentStore(db *bolt.DB) *ComputeAgentStore { + return &ComputeAgentStore{DB: db} +} + +func (c *ComputeAgentStore) Close() error { + return c.DB.Close() +} + +// GetComputeAgent returns the compute agent address of a single entry in the database for key `containerID` +// or returns an error if the key does not exist +func (c *ComputeAgentStore) GetComputeAgent(ctx context.Context, containerID string) (result string, err error) { + if err := c.DB.View(func(tx *bolt.Tx) error { + bkt := getComputeAgentBucket(tx) + if bkt == nil { + return errors.Wrapf(ErrBucketNotFound, "bucket %v", bucketKeyComputeAgent) + } + data := bkt.Get([]byte(containerID)) + if data == nil { + return errors.Wrapf(errKeyNotFound, "key %v", containerID) + } + result = string(data) + return nil + }); err != nil { + return "", err + } + + return result, nil +} + +// GetComputeAgents returns a map of the key value pairs stored in the database +// where the keys are the containerIDs and the values are the corresponding compute agent +// server addresses +func (c *ComputeAgentStore) GetComputeAgents(ctx context.Context) (map[string]string, error) { + content := map[string]string{} + if err := c.DB.View(func(tx *bolt.Tx) error { + bkt := getComputeAgentBucket(tx) + if bkt == nil { + return errors.Wrapf(ErrBucketNotFound, "bucket %v", bucketKeyComputeAgent) + } + err := bkt.ForEach(func(k, v []byte) error { + data := bkt.Get([]byte(k)) + content[string(k)] = string(data) + return nil + }) + return err + }); err != nil { + return nil, err + } + return content, nil +} + +// UpdateComputeAgent updates or adds an entry (if none already exists) to the database +// `address` corresponds to the address of the compute agent server for the `containerID` +func (c *ComputeAgentStore) UpdateComputeAgent(ctx context.Context, containerID string, address string) error { + if err := c.DB.Update(func(tx *bolt.Tx) error { + bkt, err := createComputeAgentBucket(tx) + if err != nil { + return err + } + return bkt.Put([]byte(containerID), []byte(address)) + }); err != nil { + return err + } + return nil +} + +// DeleteComputeAgent deletes an entry in the database or returns an error if none exists +// `containerID` corresponds to the target key that the entry should be deleted for +func (c *ComputeAgentStore) DeleteComputeAgent(ctx context.Context, containerID string) error { + if err := c.DB.Update(func(tx *bolt.Tx) error { + bkt := getComputeAgentBucket(tx) + if bkt == nil { + return errors.Wrapf(ErrBucketNotFound, "bucket %v", bucketKeyComputeAgent) + } + return bkt.Delete([]byte(containerID)) + }); err != nil { + return err + } + return nil +} diff --git a/ncproxystore/store_test.go b/ncproxystore/store_test.go new file mode 100644 index 0000000000..0343f6939f --- /dev/null +++ b/ncproxystore/store_test.go @@ -0,0 +1,284 @@ +package ncproxystore + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/Microsoft/hcsshim/internal/ncproxynetworking" + bolt "go.etcd.io/bbolt" +) + +func TestComputeAgentStore(t *testing.T) { + ctx := context.Background() + tempDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + db, err := bolt.Open(filepath.Join(tempDir, "networkproxy.db.test"), 0600, nil) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + store := NewComputeAgentStore(db) + containerID := "fake-container-id" + address := "123412341234" + + if err := store.UpdateComputeAgent(ctx, containerID, address); err != nil { + t.Fatal(err) + } + + actual, err := store.GetComputeAgent(ctx, containerID) + if err != nil { + t.Fatal(err) + } + + if address != actual { + t.Fatalf("compute agent addresses are not equal, expected %v but got %v", address, actual) + } + + if err := store.DeleteComputeAgent(ctx, containerID); err != nil { + t.Fatal(err) + } + + value, err := store.GetComputeAgent(ctx, containerID) + if err == nil { + t.Fatalf("expected an error, instead found value %s", value) + } +} + +func TestComputeAgentStore_GetComputeAgents(t *testing.T) { + ctx := context.Background() + tempDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + db, err := bolt.Open(filepath.Join(tempDir, "networkproxy.db.test"), 0600, nil) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + store := NewComputeAgentStore(db) + + containerIDs := []string{"fake-container-id", "fake-container-id-2"} + addresses := []string{"123412341234", "234523452345"} + + target := make(map[string]string) + for i := 0; i < len(containerIDs); i++ { + target[containerIDs[i]] = addresses[i] + if err := store.UpdateComputeAgent(ctx, containerIDs[i], addresses[i]); err != nil { + t.Fatal(err) + } + } + + actual, err := store.GetComputeAgents(ctx) + if err != nil { + t.Fatal(err) + } + + for k, v := range actual { + if target[k] != v { + t.Fatalf("expected to get %s for key %s, instead got %s", target[k], k, v) + } + } +} + +func TestEndpointStore(t *testing.T) { + ctx := context.Background() + tempDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + db, err := bolt.Open(filepath.Join(tempDir, "networkproxy.db.test"), 0600, nil) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + store := NewNetworkingStore(db) + endpointName := "test-endpoint-name" + namespaceID := "test-namespace-id" + + endpoint := &ncproxynetworking.Endpoint{ + EndpointName: endpointName, + NamespaceID: namespaceID, + } + + if err := store.CreatEndpoint(ctx, endpoint); err != nil { + t.Fatal(err) + } + + actual, err := store.GetEndpointByName(ctx, endpointName) + if err != nil { + t.Fatal(err) + } + + if actual.EndpointName != endpointName { + t.Fatalf("endpoint name is not equal, expected %v but got %v", endpointName, actual.EndpointName) + } + + if actual.NamespaceID != namespaceID { + t.Fatalf("endpoint namespace id is not equal, expected %v but got %v", namespaceID, actual.NamespaceID) + } + + if err := store.DeleteEndpoint(ctx, endpointName); err != nil { + t.Fatal(err) + } + + actual, err = store.GetEndpointByName(ctx, endpointName) + if err == nil { + t.Fatalf("expected an error, instead found endpoint %v", actual) + } +} + +func TestEndpointStore_GetAll(t *testing.T) { + ctx := context.Background() + tempDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + db, err := bolt.Open(filepath.Join(tempDir, "networkproxy.db.test"), 0600, nil) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + store := NewNetworkingStore(db) + + endpoints := []*ncproxynetworking.Endpoint{ + { + EndpointName: "endpoint-name-1", + }, + { + EndpointName: "endpoint-name-2", + }, + } + + target := make(map[string]*ncproxynetworking.Endpoint) + for i := 0; i < len(endpoints); i++ { + target[endpoints[i].EndpointName] = endpoints[i] + if err := store.CreatEndpoint(ctx, endpoints[i]); err != nil { + t.Fatal(err) + } + } + + actual, err := store.ListEndpoints(ctx) + if err != nil { + t.Fatal(err) + } + + for _, e := range actual { + endpt, ok := target[e.EndpointName] + if !ok { + t.Fatalf("unexpected endpoint with name %v found", e.EndpointName) + } + if endpt.EndpointName != e.EndpointName { + t.Fatalf("expected found endpoint to have name %v, instead found %v", endpt.EndpointName, e.EndpointName) + } + } +} + +func TestNetworkStore(t *testing.T) { + ctx := context.Background() + tempDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + db, err := bolt.Open(filepath.Join(tempDir, "networkproxy.db.test"), 0600, nil) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + store := NewNetworkingStore(db) + networkName := "test-network-name" + + network := &ncproxynetworking.Network{ + NetworkName: networkName, + } + + if err := store.CreateNetwork(ctx, network); err != nil { + t.Fatal(err) + } + + actual, err := store.GetNetworkByName(ctx, networkName) + if err != nil { + t.Fatal(err) + } + + if actual.NetworkName != networkName { + t.Fatalf("network name is not equal, expected %v but got %v", networkName, actual.NetworkName) + } + + if err := store.DeleteNetwork(ctx, networkName); err != nil { + t.Fatal(err) + } + + actual, err = store.GetNetworkByName(ctx, networkName) + if err == nil { + t.Fatalf("expected an error, instead found network %v", actual) + } +} + +func TestNetworkStore_GetAll(t *testing.T) { + ctx := context.Background() + tempDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + db, err := bolt.Open(filepath.Join(tempDir, "networkproxy.db.test"), 0600, nil) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + store := NewNetworkingStore(db) + + networks := []*ncproxynetworking.Network{ + { + NetworkName: "network-name-1", + }, + { + NetworkName: "network-name-2", + }, + } + + target := make(map[string]*ncproxynetworking.Network) + for i := 0; i < len(networks); i++ { + target[networks[i].NetworkName] = networks[i] + if err := store.CreateNetwork(ctx, networks[i]); err != nil { + t.Fatal(err) + } + } + + actual, err := store.ListNetworks(ctx) + if err != nil { + t.Fatal(err) + } + + for _, n := range actual { + network, ok := target[n.NetworkName] + if !ok { + t.Fatalf("unexpected network with name %v found", n.NetworkName) + } + if network.NetworkName != n.NetworkName { + t.Fatalf("expected found network to have name %v, instead found %v", network.NetworkName, n.NetworkName) + } + } +} diff --git a/tools/networkagent/defs.go b/tools/networkagent/defs.go index 352506fe2b..bdad12ac89 100644 --- a/tools/networkagent/defs.go +++ b/tools/networkagent/defs.go @@ -13,7 +13,7 @@ type service struct { client ncproxygrpc.NetworkConfigProxyClient containerToNamespace map[string]string endpointToNicID map[string]string - containerToNetwork map[string]string + containerToNetwork map[string][]string } type hnsSettings struct { @@ -21,8 +21,14 @@ type hnsSettings struct { IOVSettings *ncproxygrpc.IovEndpointPolicySetting `json:"iov_settings,omitempty"` } +type ncproxynetworkingSettings struct { + DeviceID string `json:"device_id,omitempty"` + VirtualFunctionIndex uint32 `json:"virtual_function_index,omitempty"` +} + type networkingSettings struct { - HNSSettings *hnsSettings `json:"hns_settings,omitempty"` + HNSSettings *hnsSettings `json:"hns_settings,omitempty"` + NCProxyNetworkingSettings *ncproxynetworkingSettings `json:"ncproxy_networking_settings,omitempty"` } type config struct { diff --git a/tools/networkagent/main.go b/tools/networkagent/main.go index 7b48d3fab2..50099e1b98 100644 --- a/tools/networkagent/main.go +++ b/tools/networkagent/main.go @@ -8,6 +8,7 @@ import ( "net" "os" "os/signal" + "strconv" "strings" "syscall" @@ -26,9 +27,8 @@ import ( var configPath = flag.String("config", "", "Path to JSON configuration file.") const ( - prefixLengthAsInt uint32 = 24 - prefixLength = "24" - ipVersion = "4" + prefixLength uint32 = 24 + ipVersion = "4" ) func generateMAC() (string, error) { @@ -70,143 +70,255 @@ func generateIPs(prefixLength string) (string, string, string) { return ipPrefixString, ipGatewayString, ipString } -func (s *service) ConfigureContainerNetworking(ctx context.Context, req *nodenetsvc.ConfigureContainerNetworkingRequest) (_ *nodenetsvc.ConfigureContainerNetworkingResponse, err error) { - // for testing purposes, make endpoints here - log.G(ctx).WithField("req", req).Info("ConfigureContainerNetworking request") +func (s *service) configureHCNNetworkingHelper(ctx context.Context, req *nodenetsvc.ConfigureContainerNetworkingRequest) (_ *nodenetsvc.ConfigureContainerNetworkingResponse, err error) { + prefixIP, gatewayIP, midIP := generateIPs(strconv.Itoa(int(prefixLength))) + addNetworkReq := &ncproxygrpc.CreateNetworkRequest{ + Network: &ncproxygrpc.Network{ + Settings: &ncproxygrpc.Network_HcnNetwork{ + HcnNetwork: &ncproxygrpc.HostComputeNetworkSettings{ + Name: req.ContainerID + "_network_hcn", + Mode: ncproxygrpc.HostComputeNetworkSettings_Transparent, + SwitchName: s.conf.NetworkingSettings.HNSSettings.SwitchName, + IpamType: ncproxygrpc.HostComputeNetworkSettings_Static, + SubnetIpaddressPrefix: []string{prefixIP}, + DefaultGateway: gatewayIP, + }, + }, + }, + } - if req.RequestType == nodenetsvc.RequestType_Setup { - prefixIP, gatewayIP, midIP := generateIPs(prefixLength) - - addNetworkReq := &ncproxygrpc.CreateNetworkRequest{ - Network: &ncproxygrpc.Network{ - Settings: &ncproxygrpc.Network_HcnNetwork{ - HcnNetwork: &ncproxygrpc.HostComputeNetworkSettings{ - Name: req.ContainerID + "_network", - Mode: ncproxygrpc.HostComputeNetworkSettings_Transparent, - SwitchName: s.conf.NetworkingSettings.HNSSettings.SwitchName, - IpamType: ncproxygrpc.HostComputeNetworkSettings_Static, - SubnetIpaddressPrefix: []string{prefixIP}, - DefaultGateway: gatewayIP, + networkResp, err := s.client.CreateNetwork(ctx, addNetworkReq) + if err != nil { + return nil, err + } + + network, err := hcn.GetNetworkByID(networkResp.ID) + if err != nil { + return nil, err + } + s.containerToNetwork[req.ContainerID] = append(s.containerToNetwork[req.ContainerID], network.Name) + + mac, err := generateMAC() + if err != nil { + return nil, err + } + + name := req.ContainerID + "_endpoint_hcn" + endpointCreateReq := &ncproxygrpc.CreateEndpointRequest{ + EndpointSettings: &ncproxygrpc.EndpointSettings{ + Settings: &ncproxygrpc.EndpointSettings_HcnEndpoint{ + HcnEndpoint: &ncproxygrpc.HcnEndpointSettings{ + Name: name, + Macaddress: mac, + Ipaddress: midIP, + IpaddressPrefixlength: prefixLength, + NetworkName: network.Name, + Policies: &ncproxygrpc.HcnEndpointPolicies{ + IovPolicySettings: s.conf.NetworkingSettings.HNSSettings.IOVSettings, }, }, }, - } + }, + } - networkResp, err := s.client.CreateNetwork(ctx, addNetworkReq) - if err != nil { - return nil, err - } + endpt, err := s.client.CreateEndpoint(ctx, endpointCreateReq) + if err != nil { + return nil, err + } - network, err := hcn.GetNetworkByID(networkResp.ID) - if err != nil { - return nil, err - } - s.containerToNetwork[req.ContainerID] = network.Name + log.G(ctx).WithField("endpt", endpt).Info("ConfigureContainerNetworking created endpoint") - mac, err := generateMAC() - if err != nil { - return nil, err - } + addEndpointReq := &ncproxygrpc.AddEndpointRequest{ + Name: name, + NamespaceID: req.NetworkNamespaceID, + } + _, err = s.client.AddEndpoint(ctx, addEndpointReq) + if err != nil { + return nil, err + } + s.containerToNamespace[req.ContainerID] = req.NetworkNamespaceID - name := req.ContainerID + "_endpoint" - endpointCreateReq := &ncproxygrpc.CreateEndpointRequest{ - EndpointSettings: &ncproxygrpc.EndpointSettings{ - Settings: &ncproxygrpc.EndpointSettings_HcnEndpoint{ - HcnEndpoint: &ncproxygrpc.HcnEndpointSettings{ - Name: name, - Macaddress: mac, - Ipaddress: midIP, - IpaddressPrefixlength: prefixLengthAsInt, - NetworkName: network.Name, - Policies: &ncproxygrpc.HcnEndpointPolicies{ - IovPolicySettings: s.conf.NetworkingSettings.HNSSettings.IOVSettings, + resultIPAddr := &nodenetsvc.ContainerIPAddress{ + Version: ipVersion, + Ip: midIP, + PrefixLength: strconv.Itoa(int(prefixLength)), + DefaultGateway: gatewayIP, + } + netInterface := &nodenetsvc.ContainerNetworkInterface{ + Name: network.Name, + MacAddress: mac, + NetworkNamespaceID: req.NetworkNamespaceID, + Ipaddresses: []*nodenetsvc.ContainerIPAddress{resultIPAddr}, + } + + return &nodenetsvc.ConfigureContainerNetworkingResponse{ + Interfaces: []*nodenetsvc.ContainerNetworkInterface{netInterface}, + }, nil +} + +func (s *service) configureNCProxyNetworkingHelper(ctx context.Context, req *nodenetsvc.ConfigureContainerNetworkingRequest) (_ *nodenetsvc.ConfigureContainerNetworkingResponse, err error) { + _, gatewayIP, midIP := generateIPs(strconv.Itoa(int(prefixLength))) + networkName := req.ContainerID + "_network_ncproxy" + addNetworkReq := &ncproxygrpc.CreateNetworkRequest{ + Network: &ncproxygrpc.Network{ + Settings: &ncproxygrpc.Network_NcproxyNetwork{ + NcproxyNetwork: &ncproxygrpc.NCProxyNetworkSettings{ + Name: networkName, + }, + }, + }, + } + + _, err = s.client.CreateNetwork(ctx, addNetworkReq) + if err != nil { + return nil, err + } + s.containerToNetwork[req.ContainerID] = append(s.containerToNetwork[req.ContainerID], networkName) + + mac, err := generateMAC() + if err != nil { + return nil, err + } + + name := req.ContainerID + "_endpoint_ncproxy" + endpointCreateReq := &ncproxygrpc.CreateEndpointRequest{ + EndpointSettings: &ncproxygrpc.EndpointSettings{ + Settings: &ncproxygrpc.EndpointSettings_NcproxyEndpoint{ + NcproxyEndpoint: &ncproxygrpc.NCProxyEndpointSettings{ + Name: name, + Macaddress: mac, + Ipaddress: midIP, + IpaddressPrefixlength: prefixLength, + NetworkName: networkName, + DefaultGateway: gatewayIP, + DeviceDetails: &ncproxygrpc.NCProxyEndpointSettings_PciDeviceDetails{ + PciDeviceDetails: &ncproxygrpc.PCIDeviceDetails{ + DeviceID: s.conf.NetworkingSettings.NCProxyNetworkingSettings.DeviceID, + VirtualFunctionIndex: s.conf.NetworkingSettings.NCProxyNetworkingSettings.VirtualFunctionIndex, }, }, }, }, - } + }, + } - endpt, err := s.client.CreateEndpoint(ctx, endpointCreateReq) - if err != nil { - return nil, err - } + endpt, err := s.client.CreateEndpoint(ctx, endpointCreateReq) + if err != nil { + return nil, err + } - log.G(ctx).WithField("endpt", endpt).Info("ConfigureContainerNetworking created endpoint") + log.G(ctx).WithField("endpt", endpt).Info("ConfigureContainerNetworking created endpoint") - addEndpointReq := &ncproxygrpc.AddEndpointRequest{ - Name: name, - NamespaceID: req.NetworkNamespaceID, - } - _, err = s.client.AddEndpoint(ctx, addEndpointReq) - if err != nil { - return nil, err - } - s.containerToNamespace[req.ContainerID] = req.NetworkNamespaceID + addEndpointReq := &ncproxygrpc.AddEndpointRequest{ + Name: name, + NamespaceID: req.NetworkNamespaceID, + } + _, err = s.client.AddEndpoint(ctx, addEndpointReq) + if err != nil { + return nil, err + } + s.containerToNamespace[req.ContainerID] = req.NetworkNamespaceID - resultIPAddr := &nodenetsvc.ContainerIPAddress{ - Version: ipVersion, - Ip: midIP, - PrefixLength: prefixLength, - DefaultGateway: gatewayIP, - } - netInterface := &nodenetsvc.ContainerNetworkInterface{ - Name: network.Name, - MacAddress: mac, - NetworkNamespaceID: req.NetworkNamespaceID, - Ipaddresses: []*nodenetsvc.ContainerIPAddress{resultIPAddr}, - } + resultIPAddr := &nodenetsvc.ContainerIPAddress{ + Version: ipVersion, + Ip: midIP, + PrefixLength: strconv.Itoa(int(prefixLength)), + DefaultGateway: gatewayIP, + } + netInterface := &nodenetsvc.ContainerNetworkInterface{ + Name: networkName, + MacAddress: mac, + NetworkNamespaceID: req.NetworkNamespaceID, + Ipaddresses: []*nodenetsvc.ContainerIPAddress{resultIPAddr}, + } - return &nodenetsvc.ConfigureContainerNetworkingResponse{ - Interfaces: []*nodenetsvc.ContainerNetworkInterface{netInterface}, - }, nil - } else if req.RequestType == nodenetsvc.RequestType_Teardown { - eReq := &ncproxygrpc.GetEndpointsRequest{} - resp, err := s.client.GetEndpoints(ctx, eReq) - if err != nil { - return nil, err - } + return &nodenetsvc.ConfigureContainerNetworkingResponse{ + Interfaces: []*nodenetsvc.ContainerNetworkInterface{netInterface}, + }, nil +} + +func (s *service) teardownConfigureContainerNetworking(ctx context.Context, req *nodenetsvc.ConfigureContainerNetworkingRequest) (_ *nodenetsvc.ConfigureContainerNetworkingResponse, err error) { + eReq := &ncproxygrpc.GetEndpointsRequest{} + resp, err := s.client.GetEndpoints(ctx, eReq) + if err != nil { + return nil, err + } - for _, endpoint := range resp.Endpoints { - if endpoint.Endpoint == nil { - log.G(ctx).WithField("id", endpoint.ID).Warn("failed to get endpoint settings") + for _, endpoint := range resp.Endpoints { + if endpoint == nil { + log.G(ctx).WithField("name", endpoint.ID).Warn("failed to find endpoint to delete") + continue + } + if endpoint.Endpoint == nil || endpoint.Endpoint.Settings == nil { + log.G(ctx).WithField("name", endpoint.ID).Warn("failed to get endpoint settings") + continue + } + if endpoint.Namespace == req.NetworkNamespaceID { + endpointName := "" + switch ep := endpoint.Endpoint.GetSettings().(type) { + case *ncproxygrpc.EndpointSettings_NcproxyEndpoint: + endpointName = ep.NcproxyEndpoint.Name + case *ncproxygrpc.EndpointSettings_HcnEndpoint: + endpointName = ep.HcnEndpoint.Name + default: + log.G(ctx).WithField("name", endpoint.ID).Warn("invalid endpoint settings type") continue } - ep := endpoint.Endpoint.GetHcnEndpoint() - if ep == nil { - log.G(ctx).WithField("name", endpoint.ID).Warn("failed to find endpoint to delete") - continue + deleteEndptReq := &ncproxygrpc.DeleteEndpointRequest{ + Name: endpointName, } - if endpoint.Namespace == req.NetworkNamespaceID { - deleteEndptReq := &ncproxygrpc.DeleteEndpointRequest{ - Name: ep.Name, - } - if _, err := s.client.DeleteEndpoint(ctx, deleteEndptReq); err != nil { - log.G(ctx).WithField("name", ep.Name).Warn("failed to delete endpoint") - } + if _, err := s.client.DeleteEndpoint(ctx, deleteEndptReq); err != nil { + log.G(ctx).WithField("name", endpointName).Warn("failed to delete endpoint") } } + } - if networkName, ok := s.containerToNetwork[req.ContainerID]; ok { + if networks, ok := s.containerToNetwork[req.ContainerID]; ok { + for _, networkName := range networks { deleteReq := &ncproxygrpc.DeleteNetworkRequest{ Name: networkName, } if _, err := s.client.DeleteNetwork(ctx, deleteReq); err != nil { log.G(ctx).WithField("name", networkName).Warn("failed to delete network") } - delete(s.containerToNetwork, req.ContainerID) } + delete(s.containerToNetwork, req.ContainerID) + } - return &nodenetsvc.ConfigureContainerNetworkingResponse{}, nil + return &nodenetsvc.ConfigureContainerNetworkingResponse{}, nil +} + +func (s *service) ConfigureContainerNetworking(ctx context.Context, req *nodenetsvc.ConfigureContainerNetworkingRequest) (_ *nodenetsvc.ConfigureContainerNetworkingResponse, err error) { + // for testing purposes, make endpoints here + log.G(ctx).WithField("req", req).Info("ConfigureContainerNetworking request") + + if req.RequestType == nodenetsvc.RequestType_Setup { + interfaces := []*nodenetsvc.ContainerNetworkInterface{} + if s.conf.NetworkingSettings.HNSSettings != nil { + result, err := s.configureHCNNetworkingHelper(ctx, req) + if err != nil { + return nil, err + } + interfaces = append(interfaces, result.Interfaces...) + } + if s.conf.NetworkingSettings.NCProxyNetworkingSettings != nil { + result, err := s.configureNCProxyNetworkingHelper(ctx, req) + if err != nil { + return nil, err + } + interfaces = append(interfaces, result.Interfaces...) + } + return &nodenetsvc.ConfigureContainerNetworkingResponse{ + Interfaces: interfaces, + }, nil + } else if req.RequestType == nodenetsvc.RequestType_Teardown { + return s.teardownConfigureContainerNetworking(ctx, req) } return nil, fmt.Errorf("invalid request type %v", req.RequestType) } func (s *service) addHelper(ctx context.Context, req *nodenetsvc.ConfigureNetworkingRequest, containerNamespaceID string) (_ *nodenetsvc.ConfigureNetworkingResponse, err error) { - return s.addHNSHelper(ctx, req, containerNamespaceID) -} - -func (s *service) addHNSHelper(ctx context.Context, req *nodenetsvc.ConfigureNetworkingRequest, containerNamespaceID string) (_ *nodenetsvc.ConfigureNetworkingResponse, err error) { eReq := &ncproxygrpc.GetEndpointsRequest{} resp, err := s.client.GetEndpoints(ctx, eReq) if err != nil { @@ -215,13 +327,12 @@ func (s *service) addHNSHelper(ctx context.Context, req *nodenetsvc.ConfigureNet log.G(ctx).WithField("endpts", resp.Endpoints).Info("ConfigureNetworking addrequest") for _, endpoint := range resp.Endpoints { - if endpoint.Endpoint == nil { - log.G(ctx).WithField("id", endpoint.ID).Warn("failed to get endpoint settings") + if endpoint == nil { + log.G(ctx).WithField("name", endpoint.ID).Warn("failed to find endpoint") continue } - ep := endpoint.Endpoint.GetHcnEndpoint() - if ep == nil { - log.G(ctx).WithField("name", endpoint.ID).Warn("failed to find endpoint") + if endpoint.Endpoint == nil || endpoint.Endpoint.Settings == nil { + log.G(ctx).WithField("name", endpoint.ID).Warn("failed to get endpoint settings") continue } if endpoint.Namespace == containerNamespaceID { @@ -230,15 +341,25 @@ func (s *service) addHNSHelper(ctx context.Context, req *nodenetsvc.ConfigureNet if err != nil { return nil, fmt.Errorf("failed to create nic GUID: %s", err) } + endpointName := "" + switch ep := endpoint.Endpoint.GetSettings().(type) { + case *ncproxygrpc.EndpointSettings_NcproxyEndpoint: + endpointName = ep.NcproxyEndpoint.Name + case *ncproxygrpc.EndpointSettings_HcnEndpoint: + endpointName = ep.HcnEndpoint.Name + default: + log.G(ctx).WithField("name", endpoint.ID).Warn("invalid endpoint settings type") + continue + } nsReq := &ncproxygrpc.AddNICRequest{ ContainerID: req.ContainerID, NicID: nicID.String(), - EndpointName: ep.Name, + EndpointName: endpointName, } if _, err := s.client.AddNIC(ctx, nsReq); err != nil { return nil, err } - s.endpointToNicID[ep.Name] = nicID.String() + s.endpointToNicID[endpointName] = nicID.String() } } @@ -259,32 +380,43 @@ func (s *service) teardownHelper(ctx context.Context, req *nodenetsvc.ConfigureN if err != nil { return nil, err } + for _, endpoint := range resp.Endpoints { - if endpoint.Endpoint == nil { - log.G(ctx).WithField("id", endpoint.ID).Warn("failed to get endpoint settings") + if endpoint == nil { + log.G(ctx).WithField("name", endpoint.ID).Warn("failed to find endpoint to delete") continue } - ep := endpoint.Endpoint.GetHcnEndpoint() - if ep == nil { - log.G(ctx).WithField("name", endpoint.ID).Warn("failed to find endpoint") + if endpoint.Endpoint == nil || endpoint.Endpoint.Settings == nil { + log.G(ctx).WithField("name", endpoint.ID).Warn("failed to get endpoint settings") continue } + if endpoint.Namespace == containerNamespaceID { - nicID, ok := s.endpointToNicID[ep.Name] + endpointName := "" + switch ep := endpoint.Endpoint.GetSettings().(type) { + case *ncproxygrpc.EndpointSettings_NcproxyEndpoint: + endpointName = ep.NcproxyEndpoint.Name + case *ncproxygrpc.EndpointSettings_HcnEndpoint: + endpointName = ep.HcnEndpoint.Name + default: + log.G(ctx).WithField("name", endpoint.ID).Warn("invalid endpoint settings type") + continue + } + nicID, ok := s.endpointToNicID[endpointName] if !ok { - log.G(ctx).WithField("name", ep.Name).Warn("endpoint was not assigned a NIC ID previously") + log.G(ctx).WithField("name", endpointName).Warn("endpoint was not assigned a NIC ID previously") continue } // remove endpoints that are in the namespace as NICs nsReq := &ncproxygrpc.DeleteNICRequest{ ContainerID: req.ContainerID, NicID: nicID, - EndpointName: ep.Name, + EndpointName: endpointName, } if _, err := s.client.DeleteNIC(ctx, nsReq); err != nil { - log.G(ctx).WithField("name", ep.Name).Warn("failed to delete endpoint nic") + log.G(ctx).WithField("name", endpointName).Warn("failed to delete endpoint nic") } - delete(s.endpointToNicID, ep.Name) + delete(s.endpointToNicID, endpointName) } } return &nodenetsvc.ConfigureNetworkingResponse{}, nil @@ -350,7 +482,7 @@ func main() { client: ncproxyClient, containerToNamespace: make(map[string]string), endpointToNicID: make(map[string]string), - containerToNetwork: make(map[string]string), + containerToNetwork: make(map[string][]string), } server := grpc.NewServer() nodenetsvc.RegisterNodeNetworkServiceServer(server, service) diff --git a/uvm/computeagent.go b/uvm/computeagent.go index 63f76380f6..54feb60dac 100644 --- a/uvm/computeagent.go +++ b/uvm/computeagent.go @@ -7,8 +7,10 @@ import ( "github.com/Microsoft/go-winio" "github.com/Microsoft/hcsshim/hcn" "github.com/Microsoft/hcsshim/internal/computeagent" + "github.com/Microsoft/hcsshim/internal/guestrequest" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/hns" + "github.com/Microsoft/hcsshim/internal/ncproxynetworking" "github.com/Microsoft/hcsshim/pkg/octtrpc" "github.com/containerd/ttrpc" "github.com/containerd/typeurl" @@ -21,6 +23,8 @@ import ( ) func init() { + typeurl.Register(&ncproxynetworking.Endpoint{}, "ncproxy/ncproxynetworking/Endpoint") + typeurl.Register(&ncproxynetworking.Network{}, "ncproxy/ncproxynetworking/Network") typeurl.Register(&hcn.HostComputeEndpoint{}, "ncproxy/hcn/HostComputeEndpoint") typeurl.Register(&hcn.HostComputeNetwork{}, "ncproxy/hcn/HostComputeNetwork") } @@ -35,8 +39,10 @@ type agentComputeSystem interface { AddEndpointToNSWithID(context.Context, string, string, *hns.HNSEndpoint) error UpdateNIC(context.Context, string, *hcsschema.NetworkAdapter) error RemoveEndpointFromNS(context.Context, string, *hns.HNSEndpoint) error - AssignDevice(context.Context, string, uint16) (*VPCIDevice, error) + AssignDevice(context.Context, string, uint16, string) (*VPCIDevice, error) RemoveDevice(context.Context, string, uint16) error + AddNICInGuest(context.Context, *guestrequest.LCOWNetworkAdapter) error + RemoveNICInGuest(context.Context, *guestrequest.LCOWNetworkAdapter) error } var _ agentComputeSystem = &UtilityVM{} @@ -63,7 +69,7 @@ func (ca *computeAgent) AssignPCI(ctx context.Context, req *computeagent.AssignP return nil, status.Error(codes.InvalidArgument, "received empty field in request") } - dev, err := ca.uvm.AssignDevice(ctx, req.DeviceID, uint16(req.VirtualFunctionIndex)) + dev, err := ca.uvm.AssignDevice(ctx, req.DeviceID, uint16(req.VirtualFunctionIndex), req.NicID) if err != nil { return nil, err } @@ -103,6 +109,18 @@ func (ca *computeAgent) AddNIC(ctx context.Context, req *computeagent.AddNICInte } switch endpt := endpoint.(type) { + case *ncproxynetworking.Endpoint: + cfg := &guestrequest.LCOWNetworkAdapter{ + NamespaceID: endpt.NamespaceID, + ID: req.NicID, + IPAddress: endpt.Settings.IPAddress, + PrefixLength: uint8(endpt.Settings.IPAddressPrefixLength), + GatewayAddress: endpt.Settings.DefaultGateway, + VPCIAssigned: true, + } + if err := ca.uvm.AddNICInGuest(ctx, cfg); err != nil { + return nil, err + } case *hcn.HostComputeEndpoint: hnsEndpoint, err := hnsGetHNSEndpointByName(endpt.Name) if err != nil { @@ -135,6 +153,8 @@ func (ca *computeAgent) ModifyNIC(ctx context.Context, req *computeagent.ModifyN } switch endpt := endpoint.(type) { + case *ncproxynetworking.Endpoint: + return nil, errors.New("modifying ncproxy networking endpoints is not supported") case *hcn.HostComputeEndpoint: hnsEndpoint, err := hnsGetHNSEndpointByName(endpt.Name) if err != nil { @@ -184,6 +204,13 @@ func (ca *computeAgent) DeleteNIC(ctx context.Context, req *computeagent.DeleteN } switch endpt := endpoint.(type) { + case *ncproxynetworking.Endpoint: + cfg := &guestrequest.LCOWNetworkAdapter{ + ID: req.NicID, + } + if err := ca.uvm.RemoveNICInGuest(ctx, cfg); err != nil { + return nil, err + } case *hcn.HostComputeEndpoint: hnsEndpoint, err := hnsGetHNSEndpointByName(endpt.Name) if err != nil { diff --git a/uvm/computeagent_test.go b/uvm/computeagent_test.go index 0f3dca44c0..aa0cd882cd 100644 --- a/uvm/computeagent_test.go +++ b/uvm/computeagent_test.go @@ -6,6 +6,7 @@ import ( "github.com/Microsoft/hcsshim/hcn" "github.com/Microsoft/hcsshim/internal/computeagent" + "github.com/Microsoft/hcsshim/internal/guestrequest" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/hns" "github.com/containerd/typeurl" @@ -28,7 +29,7 @@ func (t *testUtilityVM) UpdateNIC(ctx context.Context, id string, settings *hcss return nil } -func (t *testUtilityVM) AssignDevice(ctx context.Context, deviceID string, index uint16) (*VPCIDevice, error) { +func (t *testUtilityVM) AssignDevice(ctx context.Context, deviceID string, index uint16, vmBusGUID string) (*VPCIDevice, error) { return &VPCIDevice{}, nil } @@ -36,6 +37,14 @@ func (t *testUtilityVM) RemoveDevice(ctx context.Context, deviceID string, index return nil } +func (t *testUtilityVM) AddNICInGuest(ctx context.Context, cfg *guestrequest.LCOWNetworkAdapter) error { + return nil +} + +func (t *testUtilityVM) RemoveNICInGuest(ctx context.Context, cfg *guestrequest.LCOWNetworkAdapter) error { + return nil +} + func TestAddNIC(t *testing.T) { ctx := context.Background() diff --git a/uvm/network.go b/uvm/network.go index 7f24b13a0b..52b35a33dd 100644 --- a/uvm/network.go +++ b/uvm/network.go @@ -676,3 +676,35 @@ func (uvm *UtilityVM) UpdateNIC(ctx context.Context, id string, settings *hcssch } return uvm.modify(ctx, req) } + +// AddNICInGuest makes a request to setup a network adapter's interface inside the lcow guest. +// This is primarily used for adding NICs in the guest that have been VPCI assigned. +func (uvm *UtilityVM) AddNICInGuest(ctx context.Context, cfg *guestrequest.LCOWNetworkAdapter) error { + if !uvm.isNetworkNamespaceSupported() { + return fmt.Errorf("guest does not support network namespaces and cannot add VF NIC %+v", cfg) + } + request := hcsschema.ModifySettingRequest{} + request.GuestRequest = guestrequest.GuestRequest{ + ResourceType: guestrequest.ResourceTypeNetwork, + RequestType: requesttype.Add, + Settings: cfg, + } + + return uvm.modify(ctx, &request) +} + +// RemoveNICInGuest makes a request to remove a network interface inside the lcow guest. +// This is primarily used for removing NICs in the guest that were VPCI assigned. +func (uvm *UtilityVM) RemoveNICInGuest(ctx context.Context, cfg *guestrequest.LCOWNetworkAdapter) error { + if !uvm.isNetworkNamespaceSupported() { + return fmt.Errorf("guest does not support network namespaces and cannot remove VF NIC %+v", cfg) + } + request := hcsschema.ModifySettingRequest{} + request.GuestRequest = guestrequest.GuestRequest{ + ResourceType: guestrequest.ResourceTypeNetwork, + RequestType: requesttype.Remove, + Settings: cfg, + } + + return uvm.modify(ctx, &request) +} diff --git a/uvm/virtual_device.go b/uvm/virtual_device.go index 45e2f17031..38369b6061 100644 --- a/uvm/virtual_device.go +++ b/uvm/virtual_device.go @@ -74,13 +74,16 @@ func (vpci *VPCIDevice) Release(ctx context.Context) error { // and the VPCIDevice is returned. // Otherwise, a new request is made to assign the target device indicated by the deviceID // onto the UVM. A new VPCIDevice entry is made on the UVM and the VPCIDevice is returned -// to the caller -func (uvm *UtilityVM) AssignDevice(ctx context.Context, deviceID string, index uint16) (*VPCIDevice, error) { - guid, err := guid.NewV4() - if err != nil { - return nil, err +// to the caller. +// Allow callers to specify the vmbus guid they want the device to show up with. +func (uvm *UtilityVM) AssignDevice(ctx context.Context, deviceID string, index uint16, vmBusGUID string) (*VPCIDevice, error) { + if vmBusGUID == "" { + guid, err := guid.NewV4() + if err != nil { + return nil, err + } + vmBusGUID = guid.String() } - vmBusGUID := guid.String() key := VPCIDeviceKey{ deviceInstanceID: deviceID, From 92c42e25a425dcf821683e4569d60e13f6591d5c Mon Sep 17 00:00:00 2001 From: Kathryn Baldauf Date: Tue, 4 Jan 2022 17:03:14 -0800 Subject: [PATCH 2/3] Address PR feedback * Move ncproxy related package to unified location * Add manifest to ncproxy binary Signed-off-by: Kathryn Baldauf --- devices/drivers.go | 6 +----- devices/pnp.go | 8 ++++---- {ncproxynetworking => ncproxy/networking}/endpoints.go | 2 +- {ncproxynetworking => ncproxy/networking}/networks.go | 2 +- {ncproxystore => ncproxy/store}/buckets.go | 2 +- {ncproxystore => ncproxy/store}/store.go | 4 ++-- {ncproxystore => ncproxy/store}/store_test.go | 4 ++-- uvm/computeagent.go | 2 +- 8 files changed, 13 insertions(+), 17 deletions(-) rename {ncproxynetworking => ncproxy/networking}/endpoints.go (96%) rename {ncproxynetworking => ncproxy/networking}/networks.go (91%) rename {ncproxystore => ncproxy/store}/buckets.go (99%) rename {ncproxystore => ncproxy/store}/store.go (98%) rename {ncproxystore => ncproxy/store}/store_test.go (98%) diff --git a/devices/drivers.go b/devices/drivers.go index 2a2d01af84..804f83f092 100644 --- a/devices/drivers.go +++ b/devices/drivers.go @@ -73,11 +73,7 @@ func execModprobeInstallDriver(ctx context.Context, vm *uvm.UtilityVM, driverDir Stderr: p, } - var ( - execErr error - exitCode int - ) - exitCode, execErr = cmd.ExecInUvm(ctx, vm, req) + exitCode, execErr := cmd.ExecInUvm(ctx, vm, req) // wait to finish parsing stdout results select { diff --git a/devices/pnp.go b/devices/pnp.go index ce30bf7865..7aab42f4d1 100644 --- a/devices/pnp.go +++ b/devices/pnp.go @@ -96,10 +96,10 @@ func readCsPipeOutput(l net.Listener, errChan chan<- error, result *[]string) { errChan <- nil } -// readCsPipeOutput is a helper function that connects to a listener and reads -// the connection's comma separated output until done. resulting comma separated -// values are returned in the `result` param. The `errChan` param is used to -// propagate an errors to the calling function.f +// readAllPipeOutput is a helper function that connects to a listener and attempts to +// read the connection's entire output. Resulting output is returned as a string +// in the `result` param. The `errChan` param is used to propagate an errors to +// the calling function. func readAllPipeOutput(l net.Listener, errChan chan<- error, result *string) { defer close(errChan) c, err := l.Accept() diff --git a/ncproxynetworking/endpoints.go b/ncproxy/networking/endpoints.go similarity index 96% rename from ncproxynetworking/endpoints.go rename to ncproxy/networking/endpoints.go index e29e04824b..b8e35bd6bb 100644 --- a/ncproxynetworking/endpoints.go +++ b/ncproxy/networking/endpoints.go @@ -1,4 +1,4 @@ -package ncproxynetworking +package networking type Endpoint struct { EndpointName string diff --git a/ncproxynetworking/networks.go b/ncproxy/networking/networks.go similarity index 91% rename from ncproxynetworking/networks.go rename to ncproxy/networking/networks.go index e494c38029..7297b5a247 100644 --- a/ncproxynetworking/networks.go +++ b/ncproxy/networking/networks.go @@ -1,4 +1,4 @@ -package ncproxynetworking +package networking type Network struct { NetworkName string diff --git a/ncproxystore/buckets.go b/ncproxy/store/buckets.go similarity index 99% rename from ncproxystore/buckets.go rename to ncproxy/store/buckets.go index 69e5ffbcf1..4b2558368c 100644 --- a/ncproxystore/buckets.go +++ b/ncproxy/store/buckets.go @@ -1,4 +1,4 @@ -package ncproxystore +package store import ( bolt "go.etcd.io/bbolt" diff --git a/ncproxystore/store.go b/ncproxy/store/store.go similarity index 98% rename from ncproxystore/store.go rename to ncproxy/store/store.go index 49a4d43dce..ec4fd080e0 100644 --- a/ncproxystore/store.go +++ b/ncproxy/store/store.go @@ -1,10 +1,10 @@ -package ncproxystore +package store import ( "context" "encoding/json" - "github.com/Microsoft/hcsshim/internal/ncproxynetworking" + ncproxynetworking "github.com/Microsoft/hcsshim/internal/ncproxy/networking" "github.com/pkg/errors" bolt "go.etcd.io/bbolt" ) diff --git a/ncproxystore/store_test.go b/ncproxy/store/store_test.go similarity index 98% rename from ncproxystore/store_test.go rename to ncproxy/store/store_test.go index 0343f6939f..81b572ae53 100644 --- a/ncproxystore/store_test.go +++ b/ncproxy/store/store_test.go @@ -1,4 +1,4 @@ -package ncproxystore +package store import ( "context" @@ -7,7 +7,7 @@ import ( "path/filepath" "testing" - "github.com/Microsoft/hcsshim/internal/ncproxynetworking" + ncproxynetworking "github.com/Microsoft/hcsshim/internal/ncproxy/networking" bolt "go.etcd.io/bbolt" ) diff --git a/uvm/computeagent.go b/uvm/computeagent.go index 54feb60dac..9f748d2732 100644 --- a/uvm/computeagent.go +++ b/uvm/computeagent.go @@ -10,7 +10,7 @@ import ( "github.com/Microsoft/hcsshim/internal/guestrequest" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/hns" - "github.com/Microsoft/hcsshim/internal/ncproxynetworking" + ncproxynetworking "github.com/Microsoft/hcsshim/internal/ncproxy/networking" "github.com/Microsoft/hcsshim/pkg/octtrpc" "github.com/containerd/ttrpc" "github.com/containerd/typeurl" From 033c8946c57d2e8e9b0ead243270e066a03c1160 Mon Sep 17 00:00:00 2001 From: Kathryn Baldauf Date: Wed, 12 Jan 2022 13:55:50 -0800 Subject: [PATCH 3/3] Address PR feedback * Clean up GetEndpoints and GetNetworks calls * Use individual testing object for tests with subtests * Misc clean up Signed-off-by: Kathryn Baldauf --- ncproxy/store/store.go | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/ncproxy/store/store.go b/ncproxy/store/store.go index ec4fd080e0..678eec4409 100644 --- a/ncproxy/store/store.go +++ b/ncproxy/store/store.go @@ -11,7 +11,7 @@ import ( var ( ErrBucketNotFound = errors.New("bucket not found") - errKeyNotFound = errors.New("key does not exist") + ErrKeyNotFound = errors.New("key does not exist") ) type NetworkingStore struct { @@ -37,7 +37,7 @@ func (n *NetworkingStore) GetNetworkByName(ctx context.Context, networkName stri } data := bkt.Get([]byte(networkName)) if data == nil { - return errors.Wrapf(errKeyNotFound, "network %v", networkName) + return errors.Wrapf(ErrKeyNotFound, "network %v", networkName) } if err := json.Unmarshal(data, internalData); err != nil { return errors.Wrapf(err, "data is %v", string(data)) @@ -86,13 +86,9 @@ func (n *NetworkingStore) ListNetworks(ctx context.Context) (results []*ncproxyn return errors.Wrapf(ErrBucketNotFound, "network bucket %v", bucketKeyNetwork) } err := bkt.ForEach(func(k, v []byte) error { - data := bkt.Get([]byte(k)) - if data == nil { - return errors.Wrapf(errKeyNotFound, "network %v", k) - } internalData := &ncproxynetworking.Network{} - if err := json.Unmarshal(data, internalData); err != nil { - return errors.Wrapf(err, "data is %v", string(data)) + if err := json.Unmarshal(v, internalData); err != nil { + return errors.Wrapf(err, "data is %v", string(v)) } results = append(results, internalData) return nil @@ -114,7 +110,7 @@ func (n *NetworkingStore) GetEndpointByName(ctx context.Context, endpointName st } jsonData := bkt.Get([]byte(endpointName)) if jsonData == nil { - return errors.Wrapf(errKeyNotFound, "endpoint %v", endpointName) + return errors.Wrapf(ErrKeyNotFound, "endpoint %v", endpointName) } if err := json.Unmarshal(jsonData, endpt); err != nil { return err @@ -172,12 +168,8 @@ func (n *NetworkingStore) ListEndpoints(ctx context.Context) (results []*ncproxy return errors.Wrapf(ErrBucketNotFound, "endpoint bucket %v", bucketKeyEndpoint) } err := bkt.ForEach(func(k, v []byte) error { - jsonData := bkt.Get([]byte(k)) - if jsonData == nil { - return errors.Wrapf(errKeyNotFound, "endpoint %v", k) - } endptInternal := &ncproxynetworking.Endpoint{} - if err := json.Unmarshal(jsonData, endptInternal); err != nil { + if err := json.Unmarshal(v, endptInternal); err != nil { return err } results = append(results, endptInternal) @@ -194,28 +186,28 @@ func (n *NetworkingStore) ListEndpoints(ctx context.Context) (results []*ncproxy // ComputeAgentStore is a database that stores a key value pair of container id // to compute agent server address type ComputeAgentStore struct { - DB *bolt.DB + db *bolt.DB } func NewComputeAgentStore(db *bolt.DB) *ComputeAgentStore { - return &ComputeAgentStore{DB: db} + return &ComputeAgentStore{db: db} } func (c *ComputeAgentStore) Close() error { - return c.DB.Close() + return c.db.Close() } // GetComputeAgent returns the compute agent address of a single entry in the database for key `containerID` // or returns an error if the key does not exist func (c *ComputeAgentStore) GetComputeAgent(ctx context.Context, containerID string) (result string, err error) { - if err := c.DB.View(func(tx *bolt.Tx) error { + if err := c.db.View(func(tx *bolt.Tx) error { bkt := getComputeAgentBucket(tx) if bkt == nil { return errors.Wrapf(ErrBucketNotFound, "bucket %v", bucketKeyComputeAgent) } data := bkt.Get([]byte(containerID)) if data == nil { - return errors.Wrapf(errKeyNotFound, "key %v", containerID) + return errors.Wrapf(ErrKeyNotFound, "key %v", containerID) } result = string(data) return nil @@ -231,14 +223,13 @@ func (c *ComputeAgentStore) GetComputeAgent(ctx context.Context, containerID str // server addresses func (c *ComputeAgentStore) GetComputeAgents(ctx context.Context) (map[string]string, error) { content := map[string]string{} - if err := c.DB.View(func(tx *bolt.Tx) error { + if err := c.db.View(func(tx *bolt.Tx) error { bkt := getComputeAgentBucket(tx) if bkt == nil { return errors.Wrapf(ErrBucketNotFound, "bucket %v", bucketKeyComputeAgent) } err := bkt.ForEach(func(k, v []byte) error { - data := bkt.Get([]byte(k)) - content[string(k)] = string(data) + content[string(k)] = string(v) return nil }) return err @@ -251,7 +242,7 @@ func (c *ComputeAgentStore) GetComputeAgents(ctx context.Context) (map[string]st // UpdateComputeAgent updates or adds an entry (if none already exists) to the database // `address` corresponds to the address of the compute agent server for the `containerID` func (c *ComputeAgentStore) UpdateComputeAgent(ctx context.Context, containerID string, address string) error { - if err := c.DB.Update(func(tx *bolt.Tx) error { + if err := c.db.Update(func(tx *bolt.Tx) error { bkt, err := createComputeAgentBucket(tx) if err != nil { return err @@ -266,7 +257,7 @@ func (c *ComputeAgentStore) UpdateComputeAgent(ctx context.Context, containerID // DeleteComputeAgent deletes an entry in the database or returns an error if none exists // `containerID` corresponds to the target key that the entry should be deleted for func (c *ComputeAgentStore) DeleteComputeAgent(ctx context.Context, containerID string) error { - if err := c.DB.Update(func(tx *bolt.Tx) error { + if err := c.db.Update(func(tx *bolt.Tx) error { bkt := getComputeAgentBucket(tx) if bkt == nil { return errors.Wrapf(ErrBucketNotFound, "bucket %v", bucketKeyComputeAgent)