From 4044b238760f509805d979a2d62dfe6dd38a963b Mon Sep 17 00:00:00 2001 From: Aryan Bakliwal Date: Mon, 23 Dec 2024 00:26:31 +0530 Subject: [PATCH] add bi-direction capture support for PacketCapture Signed-off-by: Aryan Bakliwal add bidirectional capture Signed-off-by: Aryan Bakliwal update code and tests Signed-off-by: Aryan Bakliwal add enum, tests and docs Signed-off-by: Aryan Bakliwal refactor and add e2e test Signed-off-by: Aryan Bakliwal refactor Signed-off-by: Aryan Bakliwal change function and param name Signed-off-by: Aryan Bakliwal --- build/charts/antrea/crds/packetcapture.yaml | 5 +- build/yamls/antrea-aks.yml | 5 +- build/yamls/antrea-crds.yml | 5 +- build/yamls/antrea-eks.yml | 5 +- build/yamls/antrea-gke.yml | 5 +- build/yamls/antrea-ipsec.yml | 5 +- build/yamls/antrea.yml | 5 +- docs/packetcapture-guide.md | 3 + pkg/agent/packetcapture/capture/bpf.go | 230 ++++++++++++++---- pkg/agent/packetcapture/capture/bpf_test.go | 168 ++++++++++++- pkg/agent/packetcapture/capture/pcap_linux.go | 4 +- .../packetcapture/capture/pcap_unsupported.go | 2 +- pkg/agent/packetcapture/capture_interface.go | 2 +- .../packetcapture/packetcapture_controller.go | 2 +- .../packetcapture_controller_test.go | 2 +- pkg/apis/crd/v1alpha1/types.go | 11 + test/e2e/packetcapture_test.go | 51 +++- 17 files changed, 439 insertions(+), 71 deletions(-) diff --git a/build/charts/antrea/crds/packetcapture.yaml b/build/charts/antrea/crds/packetcapture.yaml index f834345b3e6..ee5288e1ddd 100644 --- a/build/charts/antrea/crds/packetcapture.yaml +++ b/build/charts/antrea/crds/packetcapture.yaml @@ -152,7 +152,10 @@ spec: type: integer minimum: 1 maximum: 65535 - + direction: + type: string + enum: ["SourceToDestination", "DestinationToSource", "Both"] + default: "SourceToDestination" timeout: type: integer minimum: 1 diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index b42e0574983..b039d50b769 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -3067,7 +3067,10 @@ spec: type: integer minimum: 1 maximum: 65535 - + direction: + type: string + enum: ["SourceToDestination", "DestinationToSource", "Both"] + default: "SourceToDestination" timeout: type: integer minimum: 1 diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml index 352079ea497..4fa419048be 100644 --- a/build/yamls/antrea-crds.yml +++ b/build/yamls/antrea-crds.yml @@ -3040,7 +3040,10 @@ spec: type: integer minimum: 1 maximum: 65535 - + direction: + type: string + enum: ["SourceToDestination", "DestinationToSource", "Both"] + default: "SourceToDestination" timeout: type: integer minimum: 1 diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 57074e9eeda..aa728788bdd 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -3067,7 +3067,10 @@ spec: type: integer minimum: 1 maximum: 65535 - + direction: + type: string + enum: ["SourceToDestination", "DestinationToSource", "Both"] + default: "SourceToDestination" timeout: type: integer minimum: 1 diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index f8fa64ad881..e086038a335 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -3067,7 +3067,10 @@ spec: type: integer minimum: 1 maximum: 65535 - + direction: + type: string + enum: ["SourceToDestination", "DestinationToSource", "Both"] + default: "SourceToDestination" timeout: type: integer minimum: 1 diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 6e70431db1d..f7a3b4bce7a 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -3067,7 +3067,10 @@ spec: type: integer minimum: 1 maximum: 65535 - + direction: + type: string + enum: ["SourceToDestination", "DestinationToSource", "Both"] + default: "SourceToDestination" timeout: type: integer minimum: 1 diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index d11446e0749..ad078160bb4 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -3067,7 +3067,10 @@ spec: type: integer minimum: 1 maximum: 65535 - + direction: + type: string + enum: ["SourceToDestination", "DestinationToSource", "Both"] + default: "SourceToDestination" timeout: type: integer minimum: 1 diff --git a/docs/packetcapture-guide.md b/docs/packetcapture-guide.md index d46379c40cc..e98b5ea04ba 100644 --- a/docs/packetcapture-guide.md +++ b/docs/packetcapture-guide.md @@ -32,6 +32,7 @@ the target traffic flow: * Destination Pod, or IP address * Transport protocol (TCP/UDP/ICMP) * Transport ports +* Direction (SourceToDestination/DestinationToSource/Both) You can start a new packet capture by creating a `PacketCapture` CR. An optional `fileServer` field can be specified to store the generated packets file. Before that, @@ -74,6 +75,8 @@ spec: pod: namespace: default name: backend + # Available options for direction: `SourceToDestination` (default), `DestinationToSource` or `Both`. + direction: SourceToDestination # optional to specify packet: ipFamily: IPv4 protocol: TCP # support arbitrary number values and string values in [TCP,UDP,ICMP] (case insensitive) diff --git a/pkg/agent/packetcapture/capture/bpf.go b/pkg/agent/packetcapture/capture/bpf.go index 841b721a25c..b8645ceca38 100644 --- a/pkg/agent/packetcapture/capture/bpf.go +++ b/pkg/agent/packetcapture/capture/bpf.go @@ -72,11 +72,74 @@ func compareProtocol(protocol uint32, skipTrue, skipFalse uint8) bpf.Instruction return bpf.JumpIf{Cond: bpf.JumpEqual, Val: protocol, SkipTrue: skipTrue, SkipFalse: skipFalse} } +func calculateSkipFalse(srcPort, dstPort uint16) uint8 { + var count uint8 + // load dstIP and compare + count += 2 + + if srcPort > 0 || dstPort > 0 { + // load fragment offset + count += 3 + + if srcPort > 0 { + count += 2 + } + if dstPort > 0 { + count += 2 + } + } + // ret keep + count += 1 + + return count +} + +// Generates IP address and port matching instructions +func compileIPPortFilter(srcAddrVal, dstAddrVal uint32, size, curLen uint8, srcPort, dstPort uint16, skipRequestCheck bool) []bpf.Instruction { + inst := []bpf.Instruction{} + + // from here we need to check the inst length to calculate skipFalse. If no protocol is set, there will be no related bpf instructions. + + // In the previous instruction, we load the packet's source IP. We then compare it with the source IP from the packet spec to determine if + // the packet is a request (from source to destination). When capturing packets in Both directions, if the source IPs do not match, + // we need to check if the packet is a response (from destination to source). In this case, we skip to the instruction where we compare the + // loaded source IP with the destination IP from the packet spec. The skipRequestCheck flag indicates whether we need to call calculateSkipFalse + // to determine how many instructions to skip before checking for response packets or just skip to the last instruction (drop packet). + if skipRequestCheck { + inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: srcAddrVal, SkipTrue: 0, SkipFalse: calculateSkipFalse(srcPort, dstPort)}) + } else { + inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: srcAddrVal, SkipTrue: 0, SkipFalse: size - curLen - uint8(len(inst)) - 2}) + } + + // dst ip + inst = append(inst, loadIPv4DestinationAddress) + inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: dstAddrVal, SkipTrue: 0, SkipFalse: size - curLen - uint8(len(inst)) - 2}) + + if srcPort > 0 || dstPort > 0 { + skipTrue := size - curLen - uint8(len(inst)) - 3 + inst = append(inst, loadIPv4HeaderOffset(skipTrue)...) + if srcPort > 0 { + inst = append(inst, loadIPv4SourcePort) + inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: uint32(srcPort), SkipTrue: 0, SkipFalse: size - curLen - uint8(len(inst)) - 2}) + } + if dstPort > 0 { + inst = append(inst, loadIPv4DestinationPort) + inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: uint32(dstPort), SkipTrue: 0, SkipFalse: size - curLen - uint8(len(inst)) - 2}) + } + + } + + // return (accept) + inst = append(inst, returnKeep) + + return inst +} + // compilePacketFilter compiles the CRD spec to bpf instructions. For now, we only focus on // ipv4 traffic. Compared to the raw BPF filter supported by libpcap, we only need to support // limited use cases, so an expression parser is not needed. -func compilePacketFilter(packetSpec *crdv1alpha1.Packet, srcIP, dstIP net.IP) []bpf.Instruction { - size := uint8(calculateInstructionsSize(packetSpec)) +func compilePacketFilter(packetSpec *crdv1alpha1.Packet, srcIP, dstIP net.IP, direction crdv1alpha1.CaptureDirection) []bpf.Instruction { + size := uint8(calculateInstructionsSize(packetSpec, direction)) // ipv4 check inst := []bpf.Instruction{loadEtherKind} @@ -101,20 +164,8 @@ func compilePacketFilter(packetSpec *crdv1alpha1.Packet, srcIP, dstIP net.IP) [] } } - // source ip - if srcIP != nil { - inst = append(inst, loadIPv4SourceAddress) - addrVal := binary.BigEndian.Uint32(srcIP[len(srcIP)-4:]) - // from here we need to check the inst length to calculate skipFalse. If no protocol is set, there will be no related bpf instructions. - inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: addrVal, SkipTrue: 0, SkipFalse: size - uint8(len(inst)) - 2}) - - } - // dst ip - if dstIP != nil { - inst = append(inst, loadIPv4DestinationAddress) - addrVal := binary.BigEndian.Uint32(dstIP[len(dstIP)-4:]) - inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: addrVal, SkipTrue: 0, SkipFalse: size - uint8(len(inst)) - 2}) - } + srcAddrVal := binary.BigEndian.Uint32(srcIP[len(srcIP)-4:]) + dstAddrVal := binary.BigEndian.Uint32(dstIP[len(dstIP)-4:]) // ports var srcPort, dstPort uint16 @@ -134,22 +185,18 @@ func compilePacketFilter(packetSpec *crdv1alpha1.Packet, srcIP, dstIP net.IP) [] } } - if srcPort > 0 || dstPort > 0 { - skipTrue := size - uint8(len(inst)) - 3 - inst = append(inst, loadIPv4HeaderOffset(skipTrue)...) - if srcPort > 0 { - inst = append(inst, loadIPv4SourcePort) - inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: uint32(srcPort), SkipTrue: 0, SkipFalse: size - uint8(len(inst)) - 2}) - } - if dstPort > 0 { - inst = append(inst, loadIPv4DestinationPort) - inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: uint32(dstPort), SkipTrue: 0, SkipFalse: size - uint8(len(inst)) - 2}) - } + inst = append(inst, loadIPv4SourceAddress) + if direction == crdv1alpha1.CaptureDirectionSourceToDestination { + inst = append(inst, compileIPPortFilter(srcAddrVal, dstAddrVal, size, uint8(len(inst)), srcPort, dstPort, false)...) + } else if direction == crdv1alpha1.CaptureDirectionDestinationToSource { + inst = append(inst, compileIPPortFilter(dstAddrVal, srcAddrVal, size, uint8(len(inst)), dstPort, srcPort, false)...) + } else { + inst = append(inst, compileIPPortFilter(srcAddrVal, dstAddrVal, size, uint8(len(inst)), srcPort, dstPort, true)...) + inst = append(inst, compileIPPortFilter(dstAddrVal, srcAddrVal, size, uint8(len(inst)), dstPort, srcPort, false)...) } - // return - inst = append(inst, returnKeep) + // return (drop) inst = append(inst, returnDrop) return inst @@ -169,50 +216,131 @@ func compilePacketFilter(packetSpec *crdv1alpha1.Packet, srcIP, dstIP net.IP) [] // (006) ld [30] # Load 4B at 30 (dest address) // (007) jeq #0x7f000001 jt 8 jf 16 # If bytes match(127.0.0.1), goto #8, else #16 // (008) ldh [20] # Load 2B at 20 (13b Fragment Offset) -// (009) jset #0x1fff jt 16 jf 10 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #10, else #16 +// (009) jset #0x1fff jt 16 jf 10 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #10, else #16 // (010) ldxb 4*([14]&0xf) # x = IP header length // (011) ldh [x + 14] # Load 2B at x+14 (TCP Source Port) -// (012) jeq #0x7b jt 13 jf 16 # TCP Source Port: If 123, goto #13, else #16 +// (012) jeq #0x7b jt 13 jf 16 # TCP Source Port: If 123, goto #13, else #16 // (013) ldh [x + 16] # Load 2B at x+16 (TCP dst port) -// (014) jeq #0x7c jt 15 jf 16 # TCP dst port: If 123, goto $15, else #16 +// (014) jeq #0x7c jt 15 jf 16 # TCP dst port: If 123, goto #15, else #16 // (015) ret #262144 # MATCH // (016) ret #0 # NOMATCH -func calculateInstructionsSize(packet *crdv1alpha1.Packet) int { +// When capturing return traffic also (i.e., both src -> dst and dst -> src), the filter might look like this: +// 'ip proto 6 and ((src host 10.244.1.2 and dst host 10.244.1.3 and src port 123 and dst port 124) or (src host 10.244.1.3 and dst host 10.244.1.2 and src port 124 and dst port 123))' +// And using `tcpdump -i '' -d` will generate the following BPF instructions: +// (000) ldh [12] # Load 2B at 12 (Ethertype) +// (001) jeq #0x800 jt 2 jf 26 # Ethertype: If IPv4, goto #2, else #26 +// (002) ldb [23] # Load 1B at 23 (IPv4 Protocol) +// (003) jeq #0x6 jt 4 jf 26 # IPv4 Protocol: If TCP, goto #4, #26 +// (004) ld [26] # Load 4B at 26 (source address) +// (005) jeq #0xaf40102 jt 6 jf 15 # If bytes match(10.244.1.2), goto #6, else #15 +// (006) ld [30] # Load 4B at 30 (dest address) +// (007) jeq #0xaf40103 jt 8 jf 26 # If bytes match(10.244.1.3), goto #8, else #26 +// (008) ldh [20] # Load 2B at 20 (13b Fragment Offset) +// (009) jset #0x1fff jt 26 jf 10 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #10, else #26 +// (010) ldxb 4*([14]&0xf) # x = IP header length +// (011) ldh [x + 14] # Load 2B at x+14 (TCP Source Port) +// (012) jeq #0x7b jt 13 jf 26 # TCP Source Port: If 123, goto #13, else #26 +// (013) ldh [x + 16] # Load 2B at x+16 (TCP dst port) +// (014) jeq #0x7c jt 25 jf 26 # TCP dst port: If 123, goto #25, else #26 +// (015) jeq #0xaf40103 jt 16 jf 26 # If bytes match(10.244.1.3), goto #16, else #26 +// (016) ld [30] # Load 4B at 30 (return traffic dest address) +// (017) jeq #0xaf40102 jt 18 jf 26 # If bytes match(10.244.1.2), goto #18, else #26 +// (018) ldh [20] # Load 2B at 20 (13b Fragment Offset) +// (019) jset #0x1fff jt 26 jf 20 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #20, else #26 +// (020) ldxb 4*([14]&0xf) # x = IP header length +// (021) ldh [x + 14] # Load 2B at x+14 (TCP Source Port) +// (022) jeq #0x7c jt 23 jf 26 # TCP Source Port: If 124, goto #23, else #26 +// (023) ldh [x + 16] # Load 2B at x+16 (TCP dst port) +// (024) jeq #0x7b jt 25 jf 26 # TCP dst port: If 123, goto #25, else #26 +// (025) ret #262144 # MATCH +// (026) ret #0 # NOMATCH + +// For simpler code generation in 'Both' direction, an extra instruction to accept the packet is added after instruction 014. +// The final instruction set looks like this: +// (000) ldh [12] # Load 2B at 12 (Ethertype) +// (001) jeq #0x800 jt 2 jf 27 # Ethertype: If IPv4, goto #2, else #27 +// (002) ldb [23] # Load 1B at 23 (IPv4 Protocol) +// (003) jeq #0x6 jt 4 jf 27 # IPv4 Protocol: If TCP, goto #4, #27 +// (004) ld [26] # Load 4B at 26 (source address) +// (005) jeq #0xaf40102 jt 6 jf 16 # If bytes match(10.244.1.2), goto #6, else #16 +// (006) ld [30] # Load 4B at 30 (dest address) +// (007) jeq #0xaf40103 jt 8 jf 27 # If bytes match(10.244.1.3), goto #8, else #27 +// (008) ldh [20] # Load 2B at 20 (13b Fragment Offset) +// (009) jset #0x1fff jt 27 jf 10 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #10, else #27 +// (010) ldxb 4*([14]&0xf) # x = IP header length +// (011) ldh [x + 14] # Load 2B at x+14 (TCP Source Port) +// (012) jeq #0x7b jt 13 jf 27 # TCP Source Port: If 123, goto #13, else #27 +// (013) ldh [x + 16] # Load 2B at x+16 (TCP dst port) +// (014) jeq #0x7c jt 15 jf 27 # TCP dst port: If 123, goto #15, else #27 +// (015) ret #262144 # MATCH +// (016) jeq #0xaf40103 jt 17 jf 27 # If bytes match(10.244.1.3), goto #17, else #27 +// (017) ld [30] # Load 4B at 30 (return traffic dest address) +// (018) jeq #0xaf40102 jt 19 jf 27 # If bytes match(10.244.1.2), goto #19, else #27 +// (019) ldh [20] # Load 2B at 20 (13b Fragment Offset) +// (020) jset #0x1fff jt 27 jf 21 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #21, else #27 +// (021) ldxb 4*([14]&0xf) # x = IP header length +// (022) ldh [x + 14] # Load 2B at x+14 (TCP Source Port) +// (023) jeq #0x7c jt 24 jf 27 # TCP Source Port: If 124, goto #24, else #27 +// (024) ldh [x + 16] # Load 2B at x+16 (TCP dst port) +// (025) jeq #0x7b jt 26 jf 27 # TCP dst port: If 123, goto #26, else #27 +// (026) ret #262144 # MATCH +// (027) ret #0 # NOMATCH + +func calculateInstructionsSize(packet *crdv1alpha1.Packet, direction crdv1alpha1.CaptureDirection) int { count := 0 // load ethertype count++ // ip check count++ + // src and dst ip + count += 4 + if packet != nil { // protocol check if packet.Protocol != nil { count += 2 } - transPort := packet.TransportHeader - if transPort.TCP != nil { - // load Fragment Offset - count += 3 - if transPort.TCP.SrcPort != nil { - count += 2 - } - if transPort.TCP.DstPort != nil { - count += 2 + transport := packet.TransportHeader + portFiltersSize := func() int { + count := 0 + if transport.TCP != nil { + // load Fragment Offset + count += 3 + if transport.TCP.SrcPort != nil { + count += 2 + } + if transport.TCP.DstPort != nil { + count += 2 + } + + } else if transport.UDP != nil { + count += 3 + if transport.UDP.SrcPort != nil { + count += 2 + } + if transport.UDP.DstPort != nil { + count += 2 + } } + return count + } + + count += portFiltersSize() - } else if transPort.UDP != nil { + if direction == crdv1alpha1.CaptureDirectionBoth { + + // extra returnKeep + count++ + + // src and dst ip (return traffic) count += 3 - if transPort.UDP.SrcPort != nil { - count += 2 - } - if transPort.UDP.DstPort != nil { - count += 2 - } + + count += portFiltersSize() + } } - // src and dst ip - count += 4 // ret command count += 2 diff --git a/pkg/agent/packetcapture/capture/bpf_test.go b/pkg/agent/packetcapture/capture/bpf_test.go index 1f911135a52..d23d513f681 100644 --- a/pkg/agent/packetcapture/capture/bpf_test.go +++ b/pkg/agent/packetcapture/capture/bpf_test.go @@ -34,9 +34,10 @@ var ( func TestCalculateInstructionsSize(t *testing.T) { tt := []struct { - name string - packet *crdv1alpha1.Packet - count int + name string + packet *crdv1alpha1.Packet + count int + direction crdv1alpha1.CaptureDirection }{ { name: "proto and host and port", @@ -50,6 +51,35 @@ func TestCalculateInstructionsSize(t *testing.T) { }, }, count: 17, + direction: crdv1alpha1.CaptureDirectionSourceToDestination, + }, + { + name: "proto and host and port and DestinationToSource", + packet: &crdv1alpha1.Packet{ + Protocol: &testTCPProtocol, + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + SrcPort: &testSrcPort, + DstPort: &testDstPort, + }, + }, + }, + count: 17, + direction: crdv1alpha1.CaptureDirectionDestinationToSource, + }, + { + name: "proto and host to port and Both", + packet: &crdv1alpha1.Packet{ + Protocol: &testTCPProtocol, + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + SrcPort: &testSrcPort, + DstPort: &testDstPort, + }, + }, + }, + count: 28, + direction: crdv1alpha1.CaptureDirectionBoth, }, { name: "proto with host", @@ -57,6 +87,7 @@ func TestCalculateInstructionsSize(t *testing.T) { Protocol: &testTCPProtocol, }, count: 10, + direction: crdv1alpha1.CaptureDirectionSourceToDestination, }, { name: "proto with src port", @@ -69,6 +100,7 @@ func TestCalculateInstructionsSize(t *testing.T) { }, }, count: 15, + direction: crdv1alpha1.CaptureDirectionSourceToDestination, }, { name: "proto with dst port", @@ -81,18 +113,20 @@ func TestCalculateInstructionsSize(t *testing.T) { }, }, count: 15, + direction: crdv1alpha1.CaptureDirectionSourceToDestination, }, { name: "any proto", packet: &crdv1alpha1.Packet{}, count: 8, + direction: crdv1alpha1.CaptureDirectionSourceToDestination, }, } for _, item := range tt { t.Run(item.name, func(t *testing.T) { - assert.Equal(t, item.count, calculateInstructionsSize(item.packet)) + assert.Equal(t, item.count, calculateInstructionsSize(item.packet, item.direction)) }) } } @@ -118,6 +152,7 @@ func TestPacketCaptureCompileBPF(t *testing.T) { DstPort: &testDstPort, }}, }, + Direction: crdv1alpha1.CaptureDirectionSourceToDestination, }, inst: []bpf.Instruction{ bpf.LoadAbsolute{Off: 12, Size: 2}, @@ -152,6 +187,7 @@ func TestPacketCaptureCompileBPF(t *testing.T) { DstPort: &testDstPort, }}, }, + Direction: crdv1alpha1.CaptureDirectionSourceToDestination, }, inst: []bpf.Instruction{ bpf.LoadAbsolute{Off: 12, Size: 2}, @@ -173,11 +209,133 @@ func TestPacketCaptureCompileBPF(t *testing.T) { bpf.RetConstant{Val: 0}, }, }, + { + name: "with-proto-port-DestinationToSource", + srcIP: net.ParseIP("127.0.0.1"), + dstIP: net.ParseIP("127.0.0.2"), + spec: &crdv1alpha1.PacketCaptureSpec{ + Packet: &crdv1alpha1.Packet{ + Protocol: &testTCPProtocol, + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + SrcPort: &testSrcPort, + DstPort: &testDstPort, + }}, + }, + Direction: crdv1alpha1.CaptureDirectionDestinationToSource, + }, + inst: []bpf.Instruction{ + bpf.LoadAbsolute{Off: 12, Size: 2}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x800, SkipFalse: 14}, + bpf.LoadAbsolute{Off: 23, Size: 1}, // ip protocol + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x6, SkipFalse: 12}, // tcp + bpf.LoadAbsolute{Off: 26, Size: 4}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x7f000002, SkipTrue: 0, SkipFalse: 10}, + bpf.LoadAbsolute{Off: 30, Size: 4}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x7f000001, SkipTrue: 0, SkipFalse: 8}, + bpf.LoadAbsolute{Off: 20, Size: 2}, // flags+fragment offset, since we need to calc where the src/dst port is + bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x1fff, SkipTrue: 6}, // do we have an L4 header? + bpf.LoadMemShift{Off: 14}, // calculate size of IP header + bpf.LoadIndirect{Off: 14, Size: 2}, // src port + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x50, SkipFalse: 3}, // port 23 + bpf.LoadIndirect{Off: 16, Size: 2}, // dst port + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x50, SkipFalse: 1}, // port 23 + bpf.RetConstant{Val: 262144}, + bpf.RetConstant{Val: 0}, + }, + }, + { + name: "with-proto-dstPort-and-Both", + srcIP: net.ParseIP("127.0.0.1"), + dstIP: net.ParseIP("127.0.0.2"), + spec: &crdv1alpha1.PacketCaptureSpec{ + Packet: &crdv1alpha1.Packet{ + Protocol: &testTCPProtocol, + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + DstPort: &testDstPort, + }}, + }, + Direction: crdv1alpha1.CaptureDirectionBoth, + }, + inst: []bpf.Instruction{ + bpf.LoadAbsolute{Off: 12, Size: 2}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x800, SkipFalse: 21}, + bpf.LoadAbsolute{Off: 23, Size: 1}, // ip protocol + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x6, SkipFalse: 19}, // tcp + bpf.LoadAbsolute{Off: 26, Size: 4}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x7f000001, SkipTrue: 0, SkipFalse: 8}, + bpf.LoadAbsolute{Off: 30, Size: 4}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x7f000002, SkipTrue: 0, SkipFalse: 15}, + bpf.LoadAbsolute{Off: 20, Size: 2}, // flags+fragment offset, since we need to calc where the src/dst port is + bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x1fff, SkipTrue: 13}, // do we have an L4 header? + bpf.LoadMemShift{Off: 14}, // calculate size of IP header + bpf.LoadIndirect{Off: 16, Size: 2}, // dst port + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x50, SkipTrue: 0, SkipFalse: 10}, // port 23 + bpf.RetConstant{Val: 262144}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x7f000002, SkipTrue: 0, SkipFalse: 8}, + bpf.LoadAbsolute{Off: 30, Size: 4}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x7f000001, SkipTrue: 0, SkipFalse: 6}, + bpf.LoadAbsolute{Off: 20, Size: 2}, // flags+fragment offset, since we need to calc where the src/dst port is + bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x1fff, SkipTrue: 4}, // do we have an L4 header? + bpf.LoadMemShift{Off: 14}, // calculate size of IP header + bpf.LoadIndirect{Off: 14, Size: 2}, // src port + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x50, SkipFalse: 1}, // port 23 + bpf.RetConstant{Val: 262144}, + bpf.RetConstant{Val: 0}, + }, + }, + { + name: "with-proto-port-and-Both", + srcIP: net.ParseIP("127.0.0.1"), + dstIP: net.ParseIP("127.0.0.2"), + spec: &crdv1alpha1.PacketCaptureSpec{ + Packet: &crdv1alpha1.Packet{ + Protocol: &testTCPProtocol, + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + SrcPort: &testSrcPort, + DstPort: &testDstPort, + }}, + }, + Direction: crdv1alpha1.CaptureDirectionBoth, + }, + inst: []bpf.Instruction{ + bpf.LoadAbsolute{Off: 12, Size: 2}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x800, SkipFalse: 25}, + bpf.LoadAbsolute{Off: 23, Size: 1}, // ip protocol + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x6, SkipFalse: 23}, // tcp + bpf.LoadAbsolute{Off: 26, Size: 4}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x7f000001, SkipTrue: 0, SkipFalse: 10}, + bpf.LoadAbsolute{Off: 30, Size: 4}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x7f000002, SkipTrue: 0, SkipFalse: 19}, + bpf.LoadAbsolute{Off: 20, Size: 2}, // flags+fragment offset, since we need to calc where the src/dst port is + bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x1fff, SkipTrue: 17}, // do we have an L4 header? + bpf.LoadMemShift{Off: 14}, // calculate size of IP header + bpf.LoadIndirect{Off: 14, Size: 2}, // src port + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x50, SkipFalse: 14}, // port 23 + bpf.LoadIndirect{Off: 16, Size: 2}, // dst port + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x50, SkipTrue: 0, SkipFalse: 12}, // port 23 + bpf.RetConstant{Val: 262144}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x7f000002, SkipTrue: 0, SkipFalse: 10}, + bpf.LoadAbsolute{Off: 30, Size: 4}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x7f000001, SkipTrue: 0, SkipFalse: 8}, + bpf.LoadAbsolute{Off: 20, Size: 2}, // flags+fragment offset, since we need to calc where the src/dst port is + bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x1fff, SkipTrue: 6}, // do we have an L4 header? + bpf.LoadMemShift{Off: 14}, // calculate size of IP header + bpf.LoadIndirect{Off: 14, Size: 2}, // src port + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x50, SkipFalse: 3}, // port 23 + bpf.LoadIndirect{Off: 16, Size: 2}, // dst port + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 0x50, SkipFalse: 1}, // port 23 + bpf.RetConstant{Val: 262144}, + bpf.RetConstant{Val: 0}, + }, + }, } for _, item := range tt { t.Run(item.name, func(t *testing.T) { - result := compilePacketFilter(item.spec.Packet, item.srcIP, item.dstIP) + result := compilePacketFilter(item.spec.Packet, item.srcIP, item.dstIP, item.spec.Direction) assert.Equal(t, item.inst, result) }) } diff --git a/pkg/agent/packetcapture/capture/pcap_linux.go b/pkg/agent/packetcapture/capture/pcap_linux.go index 8bbd4845ab8..d6aa2562566 100644 --- a/pkg/agent/packetcapture/capture/pcap_linux.go +++ b/pkg/agent/packetcapture/capture/pcap_linux.go @@ -41,9 +41,9 @@ func zeroFilter() []bpf.Instruction { return []bpf.Instruction{returnDrop} } -func (p *pcapCapture) Capture(ctx context.Context, device string, snapLen int, srcIP, dstIP net.IP, packet *crdv1alpha1.Packet) (chan gopacket.Packet, error) { +func (p *pcapCapture) Capture(ctx context.Context, device string, snapLen int, srcIP, dstIP net.IP, packet *crdv1alpha1.Packet, direction crdv1alpha1.CaptureDirection) (chan gopacket.Packet, error) { // Compile the BPF filter in advance to reduce the time window between starting the capture and applying the filter. - inst := compilePacketFilter(packet, srcIP, dstIP) + inst := compilePacketFilter(packet, srcIP, dstIP, direction) klog.V(5).InfoS("Generated bpf instructions for PacketCapture", "device", device, "srcIP", srcIP, "dstIP", dstIP, "packetSpec", packet, "bpf", inst) rawInst, err := bpf.Assemble(inst) if err != nil { diff --git a/pkg/agent/packetcapture/capture/pcap_unsupported.go b/pkg/agent/packetcapture/capture/pcap_unsupported.go index ef2cbfbcd01..acdb56fac0a 100644 --- a/pkg/agent/packetcapture/capture/pcap_unsupported.go +++ b/pkg/agent/packetcapture/capture/pcap_unsupported.go @@ -34,6 +34,6 @@ func NewPcapCapture() (*pcapCapture, error) { return nil, errors.New("PacketCapture is not implemented") } -func (p *pcapCapture) Capture(ctx context.Context, device string, snapLen int, srcIP, dstIP net.IP, packet *crdv1alpha1.Packet) (chan gopacket.Packet, error) { +func (p *pcapCapture) Capture(ctx context.Context, device string, snapLen int, srcIP, dstIP net.IP, packet *crdv1alpha1.Packet, direction crdv1alpha1.CaptureDirection) (chan gopacket.Packet, error) { return nil, errors.New("PacketCapture is not implemented") } diff --git a/pkg/agent/packetcapture/capture_interface.go b/pkg/agent/packetcapture/capture_interface.go index c607cb32c30..231be21f29f 100644 --- a/pkg/agent/packetcapture/capture_interface.go +++ b/pkg/agent/packetcapture/capture_interface.go @@ -24,5 +24,5 @@ import ( ) type PacketCapturer interface { - Capture(ctx context.Context, device string, snapLen int, srcIP, dstIP net.IP, packet *crdv1alpha1.Packet) (chan gopacket.Packet, error) + Capture(ctx context.Context, device string, snapLen int, srcIP, dstIP net.IP, packet *crdv1alpha1.Packet, direction crdv1alpha1.CaptureDirection) (chan gopacket.Packet, error) } diff --git a/pkg/agent/packetcapture/packetcapture_controller.go b/pkg/agent/packetcapture/packetcapture_controller.go index c799ffd2179..6787861481b 100644 --- a/pkg/agent/packetcapture/packetcapture_controller.go +++ b/pkg/agent/packetcapture/packetcapture_controller.go @@ -464,7 +464,7 @@ func (c *Controller) performCapture( } defer pcapngWriter.Flush() updateRateLimiter := rate.NewLimiter(rate.Every(captureStatusUpdatePeriod), 1) - packets, err := c.captureInterface.Capture(ctx, device, snapLen, srcIP, dstIP, pc.Spec.Packet) + packets, err := c.captureInterface.Capture(ctx, device, snapLen, srcIP, dstIP, pc.Spec.Packet, pc.Spec.Direction) if err != nil { return false, err } diff --git a/pkg/agent/packetcapture/packetcapture_controller_test.go b/pkg/agent/packetcapture/packetcapture_controller_test.go index 903c7815482..895a4aca9b9 100644 --- a/pkg/agent/packetcapture/packetcapture_controller_test.go +++ b/pkg/agent/packetcapture/packetcapture_controller_test.go @@ -193,7 +193,7 @@ func craftTestPacket() gopacket.Packet { type testCapture struct { } -func (p *testCapture) Capture(ctx context.Context, device string, snapLen int, srcIP, dstIP net.IP, packet *crdv1alpha1.Packet) (chan gopacket.Packet, error) { +func (p *testCapture) Capture(ctx context.Context, device string, snapLen int, srcIP, dstIP net.IP, packet *crdv1alpha1.Packet, direction crdv1alpha1.CaptureDirection) (chan gopacket.Packet, error) { ch := make(chan gopacket.Packet, testCaptureNum) for i := 0; i < 15; i++ { ch <- craftTestPacket() diff --git a/pkg/apis/crd/v1alpha1/types.go b/pkg/apis/crd/v1alpha1/types.go index e83418ef858..0c55af81bdf 100644 --- a/pkg/apis/crd/v1alpha1/types.go +++ b/pkg/apis/crd/v1alpha1/types.go @@ -442,6 +442,14 @@ type PacketCaptureFileServer struct { HostPublicKey []byte `json:"hostPublicKey,omitempty"` } +type CaptureDirection string + +const ( + CaptureDirectionSourceToDestination CaptureDirection = "SourceToDestination" + CaptureDirectionDestinationToSource CaptureDirection = "DestinationToSource" + CaptureDirectionBoth CaptureDirection = "Both" +) + type PacketCaptureSpec struct { // Timeout is the timeout for this capture session. If not specified, defaults to 60s. Timeout *int32 `json:"timeout,omitempty"` @@ -450,6 +458,9 @@ type PacketCaptureSpec struct { // for a capture session, and at least one `Pod` should be present either in the source or the destination. Source Source `json:"source"` Destination Destination `json:"destination"` + // Direction specifies which packets to capture (source -> destination, destination -> source or both). + // If not specified, defaults to SourceToDestination. + Direction CaptureDirection `json:"direction,omitempty"` // Packet defines what kind of traffic we want to capture between the source and destination. If not specified, // all kinds of traffic will count. Packet *Packet `json:"packet,omitempty"` diff --git a/test/e2e/packetcapture_test.go b/test/e2e/packetcapture_test.go index 6ac6670f342..44f66c23e73 100644 --- a/test/e2e/packetcapture_test.go +++ b/test/e2e/packetcapture_test.go @@ -174,7 +174,7 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, sftpServerIP string, p return p } - getPacketCaptureCR := func(name string, destinationPodName string, packet *crdv1alpha1.Packet, options ...packetCaptureOption) *crdv1alpha1.PacketCapture { + getPacketCaptureCR := func(name string, destinationPodName string, packet *crdv1alpha1.Packet, direction crdv1alpha1.CaptureDirection, options ...packetCaptureOption) *crdv1alpha1.PacketCapture { pc := &crdv1alpha1.PacketCapture{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -200,7 +200,8 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, sftpServerIP string, p FileServer: &crdv1alpha1.PacketCaptureFileServer{ URL: sftpURL, }, - Packet: packet, + Packet: packet, + Direction: direction, }, } for _, option := range options { @@ -220,6 +221,7 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, sftpServerIP string, p Protocol: &icmpProto, IPFamily: v1.IPv4Protocol, }, + crdv1alpha1.CaptureDirectionSourceToDestination, packetCaptureTimeout(ptr.To[int32](15)), packetCaptureFirstN(500), ), @@ -253,6 +255,7 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, sftpServerIP string, p nonExistingPodName, nonExistingPodName, nil, + crdv1alpha1.CaptureDirectionSourceToDestination, ), expectedStatus: crdv1alpha1.PacketCaptureStatus{ Conditions: []crdv1alpha1.PacketCaptureCondition{ @@ -285,6 +288,7 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, sftpServerIP string, p }, }, }, + crdv1alpha1.CaptureDirectionSourceToDestination, packetCaptureHostPublicKey(pubKey1), ), expectedStatus: crdv1alpha1.PacketCaptureStatus{ @@ -324,6 +328,7 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, sftpServerIP string, p }, }, }, + crdv1alpha1.CaptureDirectionSourceToDestination, packetCaptureHostPublicKey(pubKey2), ), expectedStatus: crdv1alpha1.PacketCaptureStatus{ @@ -358,6 +363,7 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, sftpServerIP string, p Protocol: &icmpProto, IPFamily: v1.IPv4Protocol, }, + crdv1alpha1.CaptureDirectionSourceToDestination, ), expectedStatus: crdv1alpha1.PacketCaptureStatus{ NumberCaptured: 5, @@ -392,6 +398,7 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, sftpServerIP string, p Protocol: &icmpProto, IPFamily: v1.IPv4Protocol, }, + crdv1alpha1.CaptureDirectionSourceToDestination, packetCaptureHostPublicKey(invalidPubKey.Marshal()), ), expectedStatus: crdv1alpha1.PacketCaptureStatus{ @@ -417,6 +424,46 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, sftpServerIP string, p }, }, }, + { + name: "ipv4-tcp-both", + ipVersion: 4, + pc: getPacketCaptureCR( + "ipv4-tcp-both", + tcpServerPodName, + &crdv1alpha1.Packet{ + Protocol: &tcpProto, + IPFamily: v1.IPv4Protocol, + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + DstPort: ptr.To(serverPodPort), + }, + }, + }, + crdv1alpha1.CaptureDirectionBoth, + packetCaptureHostPublicKey(pubKey1), + ), + expectedStatus: crdv1alpha1.PacketCaptureStatus{ + NumberCaptured: 10, + FilePath: getPcapURL("ipv4-tcp"), + Conditions: []crdv1alpha1.PacketCaptureCondition{ + { + Type: crdv1alpha1.PacketCaptureStarted, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Started", + }, + { + Type: crdv1alpha1.PacketCaptureComplete, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Succeed", + }, + { + Type: crdv1alpha1.PacketCaptureFileUploaded, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Succeed", + }, + }, + }, + }, } t.Run("testPacketCaptureBasic", func(t *testing.T) { for _, tc := range testcases {