user: improve pcapng writer, flush every 2s.

cli/cmd/gotls.go

@@ -39,7 +39,7 @@ var gotlsCmd = &cobra.Command{
 ecapture gotls
 ecapture gotls --elfpath=/home/cfc4n/go_https_client --hex --pid=3423
 ecapture gotls -m keylog -k /tmp/ecap_gotls_key.log --elfpath=/home/cfc4n/go_https_client -l save.log --pid=3423
-ecapture gotls -m pcap -w save_android.pcapng -i wlan0 --port 443 --elfpath=/home/cfc4n/go_https_client
+ecapture gotls -m pcap --pcapfile=save_android.pcapng -i wlan0 --port 443 --elfpath=/home/cfc4n/go_https_client
 `,
 	Run: goTLSCommandFunc,
 }

cli/cmd/tls.go

@@ -40,8 +40,8 @@ ecapture tls
 ecapture tls --hex --pid=3423
 ecapture tls -l save.log --pid=3423
 ecapture tls --libssl=/lib/x86_64-linux-gnu/libssl.so.1.1
-ecapture tls -m keylog -w save_3_0_5.pcapng --ssl_version="openssl 3.0.5" --libssl=/lib/x86_64-linux-gnu/libssl.so.3 
-ecapture tls -w pcap save_android.pcapng -i wlan0 --libssl=/apex/com.android.conscrypt/lib64/libssl.so --ssl_version="boringssl 1.1.1" --port 443
+ecapture tls -m keylog --pcapfile save_3_0_5.pcapng --ssl_version="openssl 3.0.5" --libssl=/lib/x86_64-linux-gnu/libssl.so.3 
+ecapture tls -m pcap --pcapfile save_android.pcapng -i wlan0 --libssl=/apex/com.android.conscrypt/lib64/libssl.so --ssl_version="boringssl 1.1.1" --port 443
 `,
 	Run: openSSLCommandFunc,
 }

user/module/imodule.go

@@ -220,15 +220,15 @@ func (m *Module) perfEventReader(errChan chan error, em *ebpf.Map) {
 				continue
 			}
 
-			var e event.IEventStruct
-			e, err = m.child.Decode(em, record.RawSample)
+			var event event.IEventStruct
+			event, err = m.child.Decode(em, record.RawSample)
 			if err != nil {
 				m.logger.Printf("%s\tm.child.decode error:%v", m.child.Name(), err)
 				continue
 			}
 
 			// 上报数据
-			m.Dispatcher(e)
+			m.Dispatcher(event)
 		}
 	}()
 }

user/module/probe_gotls.go

@@ -37,7 +37,6 @@ var (
 
 // GoTLSProbe represents a probe for Go SSL
 type GoTLSProbe struct {
-	Module
 	MTCProbe
 	bpfManager        *manager.Manager
 	bpfManagerOptions manager.Options
@@ -213,20 +212,6 @@ func (g *GoTLSProbe) DecodeFun(m *ebpf.Map) (event.IEventStruct, bool) {
 }
 
 func (g *GoTLSProbe) Close() error {
-
-	if g.eBPFProgramType == TlsCaptureModelTypePcap {
-		g.logger.Printf("%s\tsaving pcapng file %s\n", g.Name(), g.pcapngFilename)
-		i, err := g.savePcapng()
-		if err != nil {
-			g.logger.Printf("%s\tsave pcanNP failed, error:%v. \n", g.Name(), err)
-		}
-		if i == 0 {
-			g.logger.Printf("nothing captured, please check your network interface, see \"ecapture tls -h\" for more information.")
-		} else {
-			g.logger.Printf("%s\t save %d packets into pcapng file.\n", g.Name(), i)
-		}
-	}
-
 	g.logger.Printf("%s\tclose. \n", g.Name())
 	if err := g.bpfManager.Stop(manager.CleanAll); err != nil {
 		return fmt.Errorf("couldn't stop manager %v .", err)

user/module/probe_gotls_pcap.go

@@ -59,6 +59,11 @@ func (g *GoTLSProbe) setupManagersPcap() error {
 		return err
 	}
 
+	// Serve pcapng writer to flush pcapng file
+	go func() {
+		g.ServePcap()
+	}()
+
 	var (
 		sec string
 		fn  string

user/module/probe_openssl.go

@@ -56,7 +56,6 @@ const (
 )
 
 type MOpenSSLProbe struct {
-	Module
 	MTCProbe
 	bpfManager        *manager.Manager
 	bpfManagerOptions manager.Options
@@ -108,6 +107,8 @@ func (m *MOpenSSLProbe) Init(ctx context.Context, logger *log.Logger, conf confi
 		if err != nil {
 			return err
 		}
+		m.tcPacketsChan = make(chan *TcPacket, 2048)
+		m.tcPackets = make([]*TcPacket, 0, 256)
 		m.pcapngFilename = fileInfo
 	case config.TlsCaptureModelText:
 		fallthrough
@@ -129,7 +130,6 @@ func (m *MOpenSSLProbe) Init(ctx context.Context, logger *log.Logger, conf confi
 	m.startTime = uint64(startTime)
 	m.bootTime = uint64(bootTime)
 
-	m.tcPackets = make([]*TcPacket, 0, 1024)
 	m.tcPacketLocker = &sync.Mutex{}
 	m.masterKeyBuffer = bytes.NewBuffer([]byte{})
 
@@ -231,18 +231,6 @@ func (m *MOpenSSLProbe) start() error {
 }
 
 func (m *MOpenSSLProbe) Close() error {
-	if m.eBPFProgramType == TlsCaptureModelTypePcap {
-		m.logger.Printf("%s\tsaving pcapng file %s\n", m.Name(), m.pcapngFilename)
-		i, err := m.savePcapng()
-		if err != nil {
-			m.logger.Printf("%s\tsave pcanNP failed, error:%v. \n", m.Name(), err)
-		}
-		if i == 0 {
-			m.logger.Printf("nothing captured, please check your network interface, see \"ecapture tls -h\" for more information.")
-		} else {
-			m.logger.Printf("%s\t save %d packets into pcapng file.\n", m.Name(), i)
-		}
-	}
 
 	m.logger.Printf("%s\tclose. \n", m.Name())
 	if err := m.bpfManager.Stop(manager.CleanAll); err != nil {
@@ -411,17 +399,17 @@ func (m *MOpenSSLProbe) saveMasterSecret(secretEvent *event.MasterSecretEvent) {
 	case TlsCaptureModelTypePcap:
 		e := m.savePcapngSslKeyLog(b.Bytes())
 		if e != nil {
-			m.logger.Fatalf("%s: save CLIENT_RANDOM to pcapng error:%s", v.String(), e.Error())
+			m.logger.Fatalf("%s\t%s: save CLIENT_RANDOM to pcapng error:%s", m.Name(), v.String(), e.Error())
 			return
 		}
-		m.logger.Printf("%s: save CLIENT_RANDOM %02x to file success, %d bytes", v.String(), secretEvent.ClientRandom, b.Len())
+		m.logger.Printf("%s\t%s: save CLIENT_RANDOM %02x to file success, %d bytes", m.Name(), v.String(), secretEvent.ClientRandom, b.Len())
 	case TlsCaptureModelTypeKeylog:
 		l, e := m.keylogger.WriteString(b.String())
 		if e != nil {
-			m.logger.Fatalf("%s: save CLIENT_RANDOM to file error:%s", v.String(), e.Error())
+			m.logger.Fatalf("%s\t%s: save CLIENT_RANDOM to file error:%s", m.Name(), v.String(), e.Error())
 			return
 		}
-		m.logger.Printf("%s: save CLIENT_RANDOM %02x to file success, %d bytes", v.String(), secretEvent.ClientRandom, l)
+		m.logger.Printf("%s\t%s: save CLIENT_RANDOM %02x to file success, %d bytes", m.Name(), v.String(), secretEvent.ClientRandom, l)
 	default:
 	}
 }
@@ -445,7 +433,7 @@ func (m *MOpenSSLProbe) saveMasterSecretBSSL(secretEvent *event.MasterSecretBSSL
 		var length = int(secretEvent.HashLen)
 		if length > event.MasterSecretMaxLen {
 			length = event.MasterSecretMaxLen
-			m.logger.Println("master secret length is too long, truncate to 48 bytes, but it may cause keylog file error")
+			m.logger.Printf("%s\tmaster secret length is too long, truncate to 48 bytes, but it may cause keylog file error\n", m.Name())
 		}
 		b = bytes.NewBufferString(fmt.Sprintf("%s %02x %02x\n", hkdf.KeyLogLabelTLS12, secretEvent.ClientRandom, secretEvent.Secret[:length]))
 		m.masterKeys[k] = true
@@ -455,7 +443,7 @@ func (m *MOpenSSLProbe) saveMasterSecretBSSL(secretEvent *event.MasterSecretBSSL
 		var length int
 		length = int(secretEvent.HashLen)
 		if length > event.EvpMaxMdSize {
-			m.logger.Println("master secret length is too long, truncate to 64 bytes, but it may cause keylog file error")
+			m.logger.Printf("%s\tmaster secret length is too long, truncate to 64 bytes, but it may cause keylog file error\n", m.Name())
 			length = event.EvpMaxMdSize
 		}
 		// 判断 密钥是否为空
@@ -478,17 +466,17 @@ func (m *MOpenSSLProbe) saveMasterSecretBSSL(secretEvent *event.MasterSecretBSSL
 	case TlsCaptureModelTypePcap:
 		e := m.savePcapngSslKeyLog(b.Bytes())
 		if e != nil {
-			m.logger.Fatalf("%s: save CLIENT_RANDOM to pcapng error:%s", v.String(), e.Error())
+			m.logger.Fatalf("%s\t%s: save CLIENT_RANDOM to pcapng error:%s", m.Name(), v.String(), e.Error())
 			return
 		}
-		m.logger.Printf("%s: save CLIENT_RANDOM %02x to file success, %d bytes", v.String(), secretEvent.ClientRandom, b.Len())
+		m.logger.Printf("%s\t%s: save CLIENT_RANDOM %02x to file success, %d bytes", m.Name(), v.String(), secretEvent.ClientRandom, b.Len())
 	case TlsCaptureModelTypeKeylog:
 		l, e := m.keylogger.WriteString(b.String())
 		if e != nil {
-			m.logger.Fatalf("%s: save CLIENT_RANDOM to file error:%s", v.String(), e.Error())
+			m.logger.Fatalf("%s\t%s: save CLIENT_RANDOM to file error:%s", m.Name(), v.String(), e.Error())
 			return
 		}
-		m.logger.Printf("%s: save CLIENT_RANDOM %02x to file success, %d bytes", v.String(), secretEvent.ClientRandom, l)
+		m.logger.Printf("%s\t%s: save CLIENT_RANDOM %02x to file success, %d bytes", m.Name(), v.String(), secretEvent.ClientRandom, l)
 	default:
 	}
 }

user/module/probe_openssl_pcap.go

@@ -86,6 +86,11 @@ func (m *MOpenSSLProbe) setupManagersPcap() error {
 		return err
 	}
 
+	// Serve pcapng writer to flush pcapng file
+	go func() {
+		m.ServePcap()
+	}()
+
 	m.bpfManager = &manager.Manager{
 		Probes: []*manager.Probe{
 			// customize deleteed TC filter

user/module/probe_tc.go → user/module/probe_pcap.go

@@ -5,6 +5,7 @@ import (
 	"ecapture/pkg/util/ethernet"
 	"ecapture/user/event"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
@@ -17,6 +18,8 @@ import (
 	"time"
 )
 
+var eOverflow = errors.New("pcapNG channel overflow")
+
 // packets of TC probe
 type TcPacket struct {
 	info gopacket.CaptureInfo
@@ -68,8 +71,7 @@ func (NetCaptureData) GetSizeBytes() uint32 {
 }
 
 type MTCProbe struct {
-	//logger          *log.Logger
-	//mName           string
+	Module
 	pcapngFilename  string
 	ifIdex          int
 	ifName          string
@@ -79,6 +81,7 @@ type MTCProbe struct {
 	tcPackets       []*TcPacket
 	masterKeyBuffer *bytes.Buffer
 	tcPacketLocker  *sync.Mutex
+	tcPacketsChan   chan *TcPacket
 }
 
 func (t *MTCProbe) dumpTcSkb(tcEvent *event.TcSkbEvent) error {
@@ -146,7 +149,9 @@ func (t *MTCProbe) savePcapng() (i int, err error) {
 		return
 	}
 	t.tcPacketLocker.Lock()
-	defer t.tcPacketLocker.Unlock()
+	defer func() {
+		t.tcPacketLocker.Unlock()
+	}()
 	for _, packet := range t.tcPackets {
 		err = t.pcapWriter.WritePacket(packet.info, packet.data)
 		i++
@@ -221,7 +226,7 @@ func (t *MTCProbe) createPcapng(netIfs []net.Interface) error {
 func (t *MTCProbe) writePacket(dataLen uint32, timeStamp time.Time, packetBytes []byte) error {
 
 	// TODO add packetMeta info (e.g: process. pid, commom, etc.)
-	
+
 	info := gopacket.CaptureInfo{
 		Timestamp:     timeStamp,
 		CaptureLength: int(dataLen),
@@ -235,11 +240,70 @@ func (t *MTCProbe) writePacket(dataLen uint32, timeStamp time.Time, packetBytes
 
 	packet := &TcPacket{info: info, data: packetBytes}
 
-	t.tcPackets = append(t.tcPackets, packet)
-	return nil
+	select {
+	case t.tcPacketsChan <- packet:
+		return nil
+	default:
+		return eOverflow
+	}
 }
 
 func (t *MTCProbe) savePcapngSslKeyLog(sslKeyLog []byte) (err error) {
 	_, e := t.masterKeyBuffer.Write(sslKeyLog)
 	return e
 }
+
+// ServePcap is used to serve pcapng file
+func (t *MTCProbe) ServePcap() {
+	var ti = time.NewTicker(2 * time.Second)
+	t.logger.Printf("%s\tsaving pcapng file: %s\n", t.Name(), t.pcapngFilename)
+	var allCount int
+	defer func() {
+		if allCount == 0 {
+			t.logger.Printf("%s\tnothing captured, please check your network interface, see \"ecapture tls -h\" for more information.", t.Name())
+		} else {
+			t.logger.Printf("%s\t save %d packets into pcapng file.\n", t.Name(), allCount)
+		}
+		ti.Stop()
+	}()
+
+	var i int
+	for {
+		select {
+		case _ = <-ti.C:
+			if i == 0 || len(t.tcPackets) == 0 {
+				continue
+			}
+			n, e := t.savePcapng()
+			if e != nil {
+				t.logger.Printf("%s\tsave pcapng err:%s, maybe %d packets lost.\n", t.Name(), e.Error(), i)
+			} else {
+				t.logger.Printf("%s\tsave pcapng success, count:%d\n", t.Name(), n)
+				allCount += n
+			}
+
+			// reset counter, and reset tcPackets array
+			i = 0
+			t.tcPackets = t.tcPackets[:0]
+		case packet, ok := <-t.tcPacketsChan:
+			// append tcPackets to tcPackets Array from tcPacketsChan
+			if !ok {
+				t.logger.Printf("%s\ttcPacketsChan closed.\n", t.Name())
+			}
+			t.tcPackets = append(t.tcPackets, packet)
+			i++
+		case _ = <-t.ctx.Done():
+			if i == 0 || len(t.tcPackets) == 0 {
+				continue
+			}
+			n, e := t.savePcapng()
+			if e != nil {
+				t.logger.Printf("%s\tsave pcapng err:%s, maybe %d packets lost.\n", t.Name(), e.Error(), i)
+			} else {
+				t.logger.Printf("%s\tsave pcapng success, count:%d\n", t.Name(), n)
+				allCount += n
+			}
+			return
+		}
+	}
+}