user : Support writing pcapng files with Decryption Secrets Block (DSB).

cli/cmd/tls.go

@@ -44,6 +44,7 @@ func init() {
 	opensslCmd.PersistentFlags().StringVar(&goc.Path, "gobin", "", "path to binary built with Go toolchain.")
 	opensslCmd.PersistentFlags().StringVarP(&oc.Write, "write", "w", "", "write the  raw packets to file as pcapng format.")
 	opensslCmd.PersistentFlags().StringVarP(&oc.Ifname, "ifname", "i", "", "(TC Classifier) Interface name on which the probe will be attached.")
+	opensslCmd.PersistentFlags().Uint16Var(&oc.Port, "port", 443, "port number to capture, default:443.")
 
 	rootCmd.AddCommand(opensslCmd)
 }

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/cilium/ebpf v0.9.0
 	github.com/ehids/ebpfmanager v0.3.0
 	github.com/google/gopacket v1.1.19
+	github.com/shuLhan/go-bindata v4.0.0+incompatible
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
@@ -22,9 +23,10 @@ require (
 	github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
 	github.com/mdlayher/netlink v1.4.1 // indirect
 	github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00 // indirect
-	github.com/shuLhan/go-bindata v4.0.0+incompatible // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/vishvananda/netlink v1.1.0 // indirect
-	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
+	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 )
+
+replace github.com/google/gopacket v1.1.19 => github.com/cfc4n/gopacket v1.1.20

go.sum

@@ -1,5 +1,7 @@
 github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
 github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
+github.com/cfc4n/gopacket v1.1.20 h1:jTdmP93F+wCvLaJPk9AhwVjY2F5J4BFkk9MhXzg+5dA=
+github.com/cfc4n/gopacket v1.1.20/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
 github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/cilium/ebpf v0.9.0 h1:ldiV+FscPCQ/p3mNEV4O02EPbUZJFsoEtHvIr9xLTvk=
 github.com/cilium/ebpf v0.9.0/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY=
@@ -23,8 +25,6 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
-github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
@@ -86,8 +86,9 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -120,6 +121,7 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

kern/common.h

@@ -36,6 +36,7 @@
 #ifndef KERNEL_LESS_5_2
 const volatile u64 target_pid = 0;
 const volatile u64 target_uid = 0;
+const volatile u32 target_port = 443;
 const volatile int target_errno = BASH_ERRNO_DEFAULT;
 #else
 #endif

kern/ecapture.h

@@ -10,10 +10,12 @@
 #include "bpf/bpf_endian.h"
 
 #else
+//CO:RE is disabled
 #include <linux/kconfig.h>
 #include <uapi/linux/ptrace.h>
 #include <linux/bpf.h>
 #include <linux/socket.h>
+#include <bpf/bpf_core_read.h>
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_tracing.h>
 #include <bpf/bpf_endian.h>
@@ -22,6 +24,11 @@
 #include <linux/ip.h>
 #include <linux/in.h>
 
+struct tcphdr {
+    __be16 source;
+    __be16 dest;
+};
+
 #endif
 
 #include "common.h"

kern/openssl_tc.h

@@ -111,7 +111,7 @@ int capture_packets(struct __sk_buff *skb, bool is_ingress) {
     }
     struct tcphdr *tcp = (struct tcphdr *) (data_start + l4_hdr_off);
 
-    if (tcp->source != bpf_htons(443) && tcp->dest != bpf_htons(443)) {
+    if (tcp->source != bpf_htons(target_port) && tcp->dest != bpf_htons(target_port)) {
         return TC_ACT_OK;
     }
 
@@ -150,6 +150,5 @@ int egress_cls_func(struct __sk_buff *skb) {
 // ingress_cls_func is called for packets that are coming into the network
 SEC("classifier/ingress")
 int ingress_cls_func(struct __sk_buff *skb) {
-    return TC_ACT_OK;
     return capture_packets(skb, true);
 };

user/config_openssl.go

@@ -12,6 +12,7 @@ type OpensslConfig struct {
 	Pthread  string `json:"pthread"` // /lib/x86_64-linux-gnu/libpthread.so.0
 	Write    string `json:"write"`   // Write  the  raw  packets  to file rather than parsing and printing them out.
 	Ifname   string `json:"ifname"`  // (TC Classifier) Interface name on which the probe will be attached.
+	Port     uint16 `json:"port"`    // capture port
 	elfType  uint8  //
 }
 

user/probe_openssl.go

@@ -18,6 +18,7 @@ import (
 	"math"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 )
 
@@ -70,6 +71,8 @@ type MOpenSSLProbe struct {
 	pcapWriter        *pcapgo.NgWriter
 	startTime         uint64
 	bootTime          uint64
+	tcPackets         []*TcPacket
+	tcPacketLocker    *sync.Mutex
 }
 
 //对象初始化
@@ -113,6 +116,9 @@ func (this *MOpenSSLProbe) Init(ctx context.Context, logger *log.Logger, conf IC
 
 	this.startTime = uint64(startTime)
 	this.bootTime = uint64(bootTime)
+
+	this.tcPackets = make([]*TcPacket, 0, 1024)
+	this.tcPacketLocker = &sync.Mutex{}
 	return nil
 }
 
@@ -172,9 +178,8 @@ func (this *MOpenSSLProbe) start() error {
 }
 
 func (this *MOpenSSLProbe) Close() error {
-
 	if this.eBPFProgramType == EBPFPROGRAMTYPE_OPENSSL_TC {
-		this.logger.Printf("saving pcapng file %s\n", this.pcapngFilename)
+		this.logger.Printf("%s\tsaving pcapng file %s\n", this.Name(), this.pcapngFilename)
 		err := this.savePcapng()
 		if err != nil {
 			return err
@@ -201,6 +206,10 @@ func (this *MOpenSSLProbe) constantEditor() []manager.ConstantEditor {
 			Name:  "target_uid",
 			Value: uint64(this.conf.GetUid()),
 		},
+		{
+			Name:  "target_port",
+			Value: uint32(this.conf.(*OpensslConfig).Port),
+		},
 	}
 
 	if this.conf.GetPid() <= 0 {
@@ -272,40 +281,42 @@ func (this *MOpenSSLProbe) setupManagersUprobe() error {
 
 			// --------------------------------------------------
 			// for SSL_write_ex \ SSL_read_ex
-			{
-				Section:          "uprobe/SSL_write",
-				EbpfFuncName:     "probe_entry_SSL_write",
-				AttachToFuncName: "SSL_write_ex",
-				BinaryPath:       binaryPath,
-				UID:              "uprobe_SSL_write_ex",
-			},
-			{
-				Section:          "uretprobe/SSL_write",
-				EbpfFuncName:     "probe_ret_SSL_write",
-				AttachToFuncName: "SSL_write_ex",
-				BinaryPath:       binaryPath,
-				UID:              "uretprobe_SSL_write_ex",
-			},
-			{
-				Section:          "uprobe/SSL_read",
-				EbpfFuncName:     "probe_entry_SSL_read",
-				AttachToFuncName: "SSL_read_ex",
-				BinaryPath:       binaryPath,
-				UID:              "uprobe_SSL_read_ex",
-			},
-			{
-				Section:          "uretprobe/SSL_read",
-				EbpfFuncName:     "probe_ret_SSL_read",
-				AttachToFuncName: "SSL_read_ex",
-				BinaryPath:       binaryPath,
-				UID:              "uretprobe_SSL_read_ex",
-			},
-			{
-				Section:          "uprobe/connect",
-				EbpfFuncName:     "probe_connect",
-				AttachToFuncName: "connect",
-				BinaryPath:       libPthread,
-			},
+			/*
+				{
+					Section:          "uprobe/SSL_write",
+					EbpfFuncName:     "probe_entry_SSL_write",
+					AttachToFuncName: "SSL_write_ex",
+					BinaryPath:       binaryPath,
+					UID:              "uprobe_SSL_write_ex",
+				},
+				{
+					Section:          "uretprobe/SSL_write",
+					EbpfFuncName:     "probe_ret_SSL_write",
+					AttachToFuncName: "SSL_write_ex",
+					BinaryPath:       binaryPath,
+					UID:              "uretprobe_SSL_write_ex",
+				},
+				{
+					Section:          "uprobe/SSL_read",
+					EbpfFuncName:     "probe_entry_SSL_read",
+					AttachToFuncName: "SSL_read_ex",
+					BinaryPath:       binaryPath,
+					UID:              "uprobe_SSL_read_ex",
+				},
+				{
+					Section:          "uretprobe/SSL_read",
+					EbpfFuncName:     "probe_ret_SSL_read",
+					AttachToFuncName: "SSL_read_ex",
+					BinaryPath:       binaryPath,
+					UID:              "uretprobe_SSL_read_ex",
+				},
+				{
+					Section:          "uprobe/connect",
+					EbpfFuncName:     "probe_connect",
+					AttachToFuncName: "connect",
+					BinaryPath:       libPthread,
+				},
+			*/
 			// --------------------------------------------------
 
 			// openssl masterkey
@@ -512,7 +523,18 @@ func (this *MOpenSSLProbe) saveMasterSecret(event *MasterSecretEvent) {
 		this.logger.Fatalf("%s: save CLIENT_RANDOM to file error:%s", v.String(), e.Error())
 		return
 	}
-	this.logger.Printf("%s: save CLIENT_RANDOM %02x to file success, %d bytes", v.String(), event.ClientRandom, l)
+
+	//
+	switch this.eBPFProgramType {
+	case EBPFPROGRAMTYPE_OPENSSL_TC:
+		e = this.savePcapngSslKeyLog(b.Bytes())
+		if e != nil {
+			this.logger.Fatalf("%s: save CLIENT_RANDOM to pcapng error:%s", v.String(), e.Error())
+			return
+		}
+	default:
+		this.logger.Printf("%s: save CLIENT_RANDOM %02x to file success, %d bytes", v.String(), event.ClientRandom, l)
+	}
 }
 
 func (this *MOpenSSLProbe) Dispatcher(event event_processor.IEventStruct) {

user/probe_openssl_tc.go

@@ -15,9 +15,10 @@ import (
 	"time"
 )
 
-type netPcap struct {
-	FileObj os.File
-	Writer  pcapgo.NgWriter
+// packets of TC probe
+type TcPacket struct {
+	info gopacket.CaptureInfo
+	data []byte
 }
 
 type NetCaptureData struct {
@@ -56,15 +57,15 @@ func (this *MOpenSSLProbe) setupManagersTC() error {
 		binaryPath = "/lib/x86_64-linux-gnu/libssl.so.1.1"
 	}
 
-	this.logger.Printf("%s\tInterface:%s, Pcapng filepath:%s\n", this.Name(), ifname, this.pcapngFilename)
+	this.logger.Printf("%s\tIfname:%s, Ifindex:%d,  Port:%d, Pcapng filepath:%s\n", this.Name(), this.ifName, this.ifIdex, this.conf.(*OpensslConfig).Port, this.pcapngFilename)
 
 	// create pcapng writer
 	netIfs, err := net.Interfaces()
 	if err != nil {
 		return err
 	}
 
-	err = this.createPcapng(netIfs[1:])
+	err = this.createPcapng(netIfs)
 	if err != nil {
 		return err
 	}
@@ -137,7 +138,7 @@ func (this *MOpenSSLProbe) setupManagersTC() error {
 }
 
 func (this *MOpenSSLProbe) initDecodeFunTC() error {
-	//SSLDumpEventsMap 与解码函数映射
+	//SkbEventsMap 与解码函数映射
 	SkbEventsMap, found, err := this.bpfManager.GetMap("skb_events")
 	if err != nil {
 		return err
@@ -166,7 +167,7 @@ func (this *MOpenSSLProbe) initDecodeFunTC() error {
 
 func (this *MOpenSSLProbe) dumpTcSkb(event *TcSkbEvent) {
 
-	this.logger.Printf("%s\t%s\n", this.Name(), event.String())
+	//this.logger.Printf("%s\t%s\n", this.Name(), event.String())
 	var netEventMetadata *NetEventMetadata = &NetEventMetadata{}
 	// timeStamp is nanoseconds since system boot time
 	// To get the monotonic time since tracee was started, we have to subtract the start time from the timestamp.
@@ -180,15 +181,36 @@ func (this *MOpenSSLProbe) dumpTcSkb(event *TcSkbEvent) {
 
 // save pcapng file ,merge master key into pcapng file TODO
 func (this *MOpenSSLProbe) savePcapng() error {
+	var i int = 0
+	this.tcPacketLocker.Lock()
+	defer this.tcPacketLocker.Unlock()
+	for _, packet := range this.tcPackets {
+		err := this.pcapWriter.WritePacket(packet.info, packet.data)
+		i++
+		if err != nil {
+			return err
+		}
+	}
+	this.logger.Printf("%s\t save %d packets into pcapng file.\n", this.Name(), i)
 	return this.pcapWriter.Flush()
 }
 
 func (this *MOpenSSLProbe) createPcapng(netIfs []net.Interface) error {
-	pcapFile, err := os.OpenFile(this.pcapngFilename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
+	pcapFile, err := os.OpenFile(this.pcapngFilename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644)
 	if err != nil {
 		return fmt.Errorf("error creating pcap file: %v", err)
 	}
 
+	// TODO : write Application "ecapture.lua" to decode PID/Comm info.
+	pcapOption := pcapgo.NgWriterOptions{
+		SectionInfo: pcapgo.NgSectionInfo{
+			Hardware:    "eCapture Hardware",
+			OS:          "",
+			Application: "ecapture.lua",
+			Comment:     "see https://ecapture.cc for more information. CFC4N <cfc4n@cnxct.com>",
+		},
+	}
+	// write interface description
 	ngIface := pcapgo.NgInterface{
 		Name:       this.conf.(*OpensslConfig).Ifname,
 		Comment:    "eCapture (旁观者): github.com/ehids/ecapture",
@@ -197,7 +219,7 @@ func (this *MOpenSSLProbe) createPcapng(netIfs []net.Interface) error {
 		SnapLength: uint32(math.MaxUint16),
 	}
 
-	pcapWriter, err := pcapgo.NewNgWriterInterface(pcapFile, ngIface, pcapgo.NgWriterOptions{})
+	pcapWriter, err := pcapgo.NewNgWriterInterface(pcapFile, ngIface, pcapOption)
 	if err != nil {
 		return err
 	}
@@ -217,6 +239,7 @@ func (this *MOpenSSLProbe) createPcapng(netIfs []net.Interface) error {
 			return err
 		}
 	}
+
 	// Flush the header
 	err = pcapWriter.Flush()
 	if err != nil {
@@ -236,10 +259,13 @@ func (this *MOpenSSLProbe) writePacket(dataLen uint32, ifaceIdx int, timeStamp t
 		InterfaceIndex: ifaceIdx,
 	}
 
-	// TODO 按照进程PID方式，划分独立的Writer
-	err := this.pcapWriter.WritePacket(info, packetBytes)
-	if err != nil {
-		return err
-	}
+	packet := &TcPacket{info: info, data: packetBytes}
+
+	this.tcPackets = append(this.tcPackets, packet)
 	return nil
 }
+
+func (this *MOpenSSLProbe) savePcapngSslKeyLog(sslKeyLog []byte) (err error) {
+	err = this.pcapWriter.WriteDecryptionSecretsBlock(pcapgo.DSB_SECRETS_TYPE_TLS, sslKeyLog)
+	return
+}