From b5f45db2e9d667f0a7567e2955539a972af7ce65 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Mon, 6 Mar 2023 23:27:01 +0800 Subject: [PATCH 1/8] kern: rename openssl_tc.h to tc.h Signed-off-by: CFC4N --- kern/common.h | 4 ++-- kern/openssl.h | 2 +- kern/{openssl_tc.h => tc.h} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename kern/{openssl_tc.h => tc.h} (100%) diff --git a/kern/common.h b/kern/common.h index a0c407f7b..55b170612 100644 --- a/kern/common.h +++ b/kern/common.h @@ -40,14 +40,14 @@ #define SA_DATA_LEN 14 #define BASH_ERRNO_DEFAULT 128 -///////// for TC & XDP ebpf programs in openssl_tc.h +///////// for TC & XDP ebpf programs in tc.h #define TC_ACT_OK 0 #define ETH_P_IP 0x0800 /* Internet Protocol packet */ #define SKB_MAX_DATA_SIZE 2048 // .rodata section bug via : https://github.com/gojue/ecapture/issues/39 #ifndef KERNEL_LESS_5_2 -// alawyse, we used it in openssl_tc.h +// alawyse, we used it in tc.h const volatile u64 target_port = 443; // Optional Target PID and UID diff --git a/kern/openssl.h b/kern/openssl.h index 64c25c345..aa667dc2d 100644 --- a/kern/openssl.h +++ b/kern/openssl.h @@ -13,7 +13,7 @@ // limitations under the License. #include "ecapture.h" -#include "openssl_tc.h" +#include "tc.h" enum ssl_data_event_type { kSSLRead, kSSLWrite }; const u32 invalidFD = 0; diff --git a/kern/openssl_tc.h b/kern/tc.h similarity index 100% rename from kern/openssl_tc.h rename to kern/tc.h From 367a62fc81e18b9f50085e8ab6dea092b230385e Mon Sep 17 00:00:00 2001 From: CFC4N Date: Mon, 6 Mar 2023 23:29:51 +0800 Subject: [PATCH 2/8] user: add gotls TC model to capture SKB events add master secret events. Signed-off-by: CFC4N --- kern/gotls_kern.c | 1 + user/module/probe_gotls.go | 60 +++++++++++-- user/module/probe_gotls_tc.go | 153 ++++++++++++++++++++++++++++++++ user/module/probe_openssl.go | 85 ++++-------------- user/module/probe_openssl_tc.go | 126 -------------------------- user/module/probe_tc.go | 147 ++++++++++++++++++++++++++++++ 6 files changed, 368 insertions(+), 204 deletions(-) create mode 100644 user/module/probe_gotls_tc.go create mode 100644 user/module/probe_tc.go diff --git a/kern/gotls_kern.c b/kern/gotls_kern.c index 24c5ef8fb..340f8d733 100644 --- a/kern/gotls_kern.c +++ b/kern/gotls_kern.c @@ -15,6 +15,7 @@ /* Copyright © 2022 Hengqi Chen */ #include "ecapture.h" #include "gotls.h" +#include "tc.h" struct go_tls_event { u64 ts_ns; diff --git a/user/module/probe_gotls.go b/user/module/probe_gotls.go index 4d68a113b..c2edfcd6c 100644 --- a/user/module/probe_gotls.go +++ b/user/module/probe_gotls.go @@ -21,12 +21,21 @@ func init() { Register(mod) } +const ( + goTlsHookFunc = "crypto/tls.(*Conn).writeRecordLocked" + goTlsMasterKeyFunc = "crypto/tls.(*Config).writeKeyLog" +) + // GoTLSProbe represents a probe for Go SSL type GoTLSProbe struct { Module - mngr *manager.Manager - path string - isRegisterABI bool + MTCProbe + bpfManager *manager.Manager + bpfManagerOptions manager.Options + eventFuncMaps map[*ebpf.Map]event.IEventStruct + eventMaps []*ebpf.Map + path string + isRegisterABI bool } func (this *GoTLSProbe) Init(ctx context.Context, l *log.Logger, cfg config.IConfig) error { @@ -34,6 +43,8 @@ func (this *GoTLSProbe) Init(ctx context.Context, l *log.Logger, cfg config.ICon this.conf = cfg this.Module.SetChild(this) + this.eventMaps = make([]*ebpf.Map, 0, 2) + this.eventFuncMaps = make(map[*ebpf.Map]event.IEventStruct) this.path = cfg.(*config.GoTLSConfig).Path ver, err := proc.ExtraceGoVersion(this.path) if err != nil { @@ -65,12 +76,12 @@ func (this *GoTLSProbe) Start() error { fn = "gotls_text_stack" } this.logger.Printf("%s\teBPF Function Name:%s, isRegisterABI:%t\n", this.Name(), fn, this.isRegisterABI) - this.mngr = &manager.Manager{ + this.bpfManager = &manager.Manager{ Probes: []*manager.Probe{ { Section: sec, EbpfFuncName: fn, - AttachToFuncName: "crypto/tls.(*Conn).writeRecordLocked", + AttachToFuncName: goTlsHookFunc, BinaryPath: this.path, }, @@ -108,17 +119,50 @@ func (this *GoTLSProbe) Start() error { Max: math.MaxUint64, }, } - if err := this.mngr.InitWithOptions(bytes.NewReader(byteBuf), opts); err != nil { + if err := this.bpfManager.InitWithOptions(bytes.NewReader(byteBuf), opts); err != nil { return err } - return this.mngr.Start() + return this.bpfManager.Start() +} + +// 通过elf的常量替换方式传递数据 +func (this *GoTLSProbe) constantEditor() []manager.ConstantEditor { + var editor = []manager.ConstantEditor{ + { + Name: "target_pid", + Value: uint64(this.conf.GetPid()), + //FailOnMissing: true, + }, + { + Name: "target_uid", + Value: uint64(this.conf.GetUid()), + }, + { + Name: "target_port", + Value: uint64(this.conf.(*config.OpensslConfig).Port), + }, + } + + if this.conf.GetPid() <= 0 { + this.logger.Printf("%s\ttarget all process. \n", this.Name()) + } else { + this.logger.Printf("%s\ttarget PID:%d \n", this.Name(), this.conf.GetPid()) + } + + if this.conf.GetUid() <= 0 { + this.logger.Printf("%s\ttarget all users. \n", this.Name()) + } else { + this.logger.Printf("%s\ttarget UID:%d \n", this.Name(), this.conf.GetUid()) + } + + return editor } func (this *GoTLSProbe) Events() []*ebpf.Map { var maps []*ebpf.Map - m, ok, err := this.mngr.GetMap("events") + m, ok, err := this.bpfManager.GetMap("events") if err != nil || !ok { return maps } diff --git a/user/module/probe_gotls_tc.go b/user/module/probe_gotls_tc.go new file mode 100644 index 000000000..413afaf29 --- /dev/null +++ b/user/module/probe_gotls_tc.go @@ -0,0 +1,153 @@ +// Copyright 2022 CFC4N . All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package module + +import ( + "ecapture/user/config" + "ecapture/user/event" + "errors" + "fmt" + "github.com/cilium/ebpf" + manager "github.com/gojue/ebpfmanager" + "golang.org/x/sys/unix" + "math" + "net" +) + +func (this *GoTLSProbe) setupManagersTC() error { + var ifname string + + ifname = this.conf.(*config.GoTLSConfig).Ifname + this.ifName = ifname + interf, err := net.InterfaceByName(this.ifName) + if err != nil { + return err + } + + // loopback devices are special, some tc probes should be skipped + isNetIfaceLo := interf.Flags&net.FlagLoopback == net.FlagLoopback + skipLoopback := true // TODO: detect loopback devices via aquasecrity/tracee/pkg/ebpf/probes/probe.go line 322 + if isNetIfaceLo && skipLoopback { + return fmt.Errorf("%s\t%s is a loopback interface, skip it", this.Name(), this.ifName) + } + this.ifIdex = interf.Index + + this.logger.Printf("%s\tHOOK type:golang elf, binrayPath:%s\n", this.Name(), this.path) + this.logger.Printf("%s\tIfname:%s, Ifindex:%d, Port:%d, Pcapng filepath:%s\n", this.Name(), this.ifName, this.ifIdex, this.conf.(*config.GoTLSConfig).Port, this.pcapngFilename) + this.logger.Printf("%s\tHook masterKey function:%s\n", this.Name(), goTlsMasterKeyFunc) + + // create pcapng writer + netIfs, err := net.Interfaces() + if err != nil { + return err + } + + err = this.createPcapng(netIfs) + if err != nil { + return err + } + + this.bpfManager = &manager.Manager{ + Probes: []*manager.Probe{ + { + Section: "classifier/egress", + EbpfFuncName: "egress_cls_func", + Ifname: this.ifName, + NetworkDirection: manager.Egress, + }, + { + Section: "classifier/ingress", + EbpfFuncName: "ingress_cls_func", + Ifname: this.ifName, + NetworkDirection: manager.Ingress, + }, + // -------------------------------------------------- + + // gotls master secrets + { + // TODO + Section: "uprobe/gotls_writekeylog", + EbpfFuncName: "probe_gotls_writekeylog", + AttachToFuncName: goTlsMasterKeyFunc, + BinaryPath: this.path, + UID: "uprobe_gotls_master_key", + }, + }, + + Maps: []*manager.Map{ + { + Name: "mastersecret_events", + }, + { + Name: "skb_events", + }, + }, + } + + this.bpfManagerOptions = manager.Options{ + DefaultKProbeMaxActive: 512, + + VerifierOptions: ebpf.CollectionOptions{ + Programs: ebpf.ProgramOptions{ + LogSize: 2097152, + }, + }, + + RLimit: &unix.Rlimit{ + Cur: math.MaxUint64, + Max: math.MaxUint64, + }, + } + + if this.conf.EnableGlobalVar() { + // 填充 RewriteContants 对应map + this.bpfManagerOptions.ConstantEditors = this.constantEditor() + } + return nil +} + +func (this *GoTLSProbe) initDecodeFunTC() error { + //SkbEventsMap 与解码函数映射 + SkbEventsMap, found, err := this.bpfManager.GetMap("skb_events") + if err != nil { + return err + } + if !found { + return errors.New("cant found map:skb_events") + } + this.eventMaps = append(this.eventMaps, SkbEventsMap) + sslEvent := &event.TcSkbEvent{} + //sslEvent.SetModule(this) + this.eventFuncMaps[SkbEventsMap] = sslEvent + + // TODO write a master secrets map at ebpf code + MasterkeyEventsMap, found, err := this.bpfManager.GetMap("mastersecret_events") + if err != nil { + return err + } + if !found { + return errors.New("cant found map:mastersecret_events") + } + this.eventMaps = append(this.eventMaps, MasterkeyEventsMap) + + var masterkeyEvent event.IEventStruct + + // TODO goTLS Event struct + masterkeyEvent = &event.MasterSecretEvent{} + + //masterkeyEvent.SetModule(this) + this.eventFuncMaps[MasterkeyEventsMap] = masterkeyEvent + return nil +} diff --git a/user/module/probe_openssl.go b/user/module/probe_openssl.go index ab7041839..f27dcd78e 100644 --- a/user/module/probe_openssl.go +++ b/user/module/probe_openssl.go @@ -26,7 +26,6 @@ import ( "fmt" "github.com/cilium/ebpf" manager "github.com/gojue/ebpfmanager" - "github.com/google/gopacket/pcapgo" "golang.org/x/sys/unix" "log" "math" @@ -58,27 +57,19 @@ const ( type MOpenSSLProbe struct { Module + MTCProbe bpfManager *manager.Manager bpfManagerOptions manager.Options eventFuncMaps map[*ebpf.Map]event.IEventStruct eventMaps []*ebpf.Map // pid[fd:Addr] - pidConns map[uint32]map[uint32]string + //pidConns map[uint32]map[uint32]string keyloggerFilename string keylogger *os.File masterKeys map[string]bool eBPFProgramType EBPFPROGRAMTYPE - pcapngFilename string - ifIdex int - ifName string - pcapWriter *pcapgo.NgWriter - startTime uint64 - bootTime uint64 - tcPackets []*TcPacket - masterKeyBuffer *bytes.Buffer - tcPacketLocker *sync.Mutex sslVersionBpfMap map[string]string // bpf map key: ssl version, value: bpf map key sslBpfFile string // ssl bpf file @@ -93,7 +84,7 @@ func (this *MOpenSSLProbe) Init(ctx context.Context, logger *log.Logger, conf co this.Module.SetChild(this) this.eventMaps = make([]*ebpf.Map, 0, 2) this.eventFuncMaps = make(map[*ebpf.Map]event.IEventStruct) - this.pidConns = make(map[uint32]map[uint32]string) + //this.pidConns = make(map[uint32]map[uint32]string) this.masterKeys = make(map[string]bool) this.sslVersionBpfMap = make(map[string]string) @@ -227,14 +218,18 @@ func (this *MOpenSSLProbe) start() error { func (this *MOpenSSLProbe) Close() error { if this.eBPFProgramType == EbpfprogramtypeOpensslTc { this.logger.Printf("%s\tsaving pcapng file %s\n", this.Name(), this.pcapngFilename) - err := this.savePcapng() + i, err := this.savePcapng() if err != nil { - return err + this.logger.Printf("%s\tsave pcanNP failed, error:%v. \n", this.Name(), err) + } + if i == 0 { + this.logger.Printf("nothing captured, please check your network interface, see \"ecapture tls -h\" for more information.") + } else { + this.logger.Printf("%s\t save %d packets into pcapng file.\n", this.Name(), i) } } this.logger.Printf("%s\tclose. \n", this.Name()) - if err := this.bpfManager.Stop(manager.CleanAll); err != nil { return fmt.Errorf("couldn't stop manager %v .", err) } @@ -474,59 +469,6 @@ func (this *MOpenSSLProbe) Events() []*ebpf.Map { return this.eventMaps } -func (this *MOpenSSLProbe) AddConn(pid, fd uint32, addr string) { - // save to map - var m map[uint32]string - var f bool - m, f = this.pidConns[pid] - if !f { - m = make(map[uint32]string) - } - m[fd] = addr - this.pidConns[pid] = m - return -} - -// process exit :fd is 0 , delete all pid map -// fd exit :pid > 0, fd > 0, delete fd value -// TODO add fd * pid exit event hook -func (this *MOpenSSLProbe) DelConn(pid, fd uint32) { - // delete from map - if pid == 0 { - return - } - - if fd == 0 { - delete(this.pidConns, pid) - } - - var m map[uint32]string - var f bool - m, f = this.pidConns[pid] - if !f { - return - } - delete(m, fd) - this.pidConns[pid] = m - return -} - -func (this *MOpenSSLProbe) GetConn(pid, fd uint32) string { - addr := "" - var m map[uint32]string - var f bool - m, f = this.pidConns[pid] - if !f { - return ConnNotFound - } - - addr, f = m[fd] - if !f { - return ConnNotFound - } - return addr -} - func (this *MOpenSSLProbe) saveMasterSecret(secretEvent *event.MasterSecretEvent) { var k = fmt.Sprintf("%02x", secretEvent.ClientRandom) @@ -709,13 +651,16 @@ func (this *MOpenSSLProbe) Dispatcher(eventStruct event.IEventStruct) { // detect eventStruct type switch eventStruct.(type) { case *event.ConnDataEvent: - this.AddConn(eventStruct.(*event.ConnDataEvent).Pid, eventStruct.(*event.ConnDataEvent).Fd, eventStruct.(*event.ConnDataEvent).Addr) + //this.AddConn(eventStruct.(*event.ConnDataEvent).Pid, eventStruct.(*event.ConnDataEvent).Fd, eventStruct.(*event.ConnDataEvent).Addr) case *event.MasterSecretEvent: this.saveMasterSecret(eventStruct.(*event.MasterSecretEvent)) case *event.MasterSecretBSSLEvent: this.saveMasterSecretBSSL(eventStruct.(*event.MasterSecretBSSLEvent)) case *event.TcSkbEvent: - this.dumpTcSkb(eventStruct.(*event.TcSkbEvent)) + err := this.dumpTcSkb(eventStruct.(*event.TcSkbEvent)) + if err != nil { + this.logger.Printf("%s\t save packet error %s .\n", this.Name(), err.Error()) + } } //this.logger.Println(eventStruct) } diff --git a/user/module/probe_openssl_tc.go b/user/module/probe_openssl_tc.go index 06c70c8c8..471396290 100644 --- a/user/module/probe_openssl_tc.go +++ b/user/module/probe_openssl_tc.go @@ -21,32 +21,12 @@ import ( "fmt" "github.com/cilium/ebpf" manager "github.com/gojue/ebpfmanager" - "github.com/google/gopacket" - "github.com/google/gopacket/layers" - "github.com/google/gopacket/pcapgo" "golang.org/x/sys/unix" "math" "net" - "os" "strings" - "time" ) -// packets of TC probe -type TcPacket struct { - info gopacket.CaptureInfo - data []byte -} - -type NetCaptureData struct { - PacketLength uint32 `json:"pktLen"` - ConfigIfaceIndex uint32 `json:"ifIndex"` -} - -func (NetCaptureData) GetSizeBytes() uint32 { - return 8 -} - type NetEventMetadata struct { TimeStamp uint64 `json:"timeStamp"` HostTid uint32 `json:"hostTid"` @@ -208,109 +188,3 @@ func (this *MOpenSSLProbe) initDecodeFunTC() error { this.eventFuncMaps[MasterkeyEventsMap] = masterkeyEvent return nil } - -func (this *MOpenSSLProbe) dumpTcSkb(tcEvent *event.TcSkbEvent) { - var timeStamp = this.bootTime + tcEvent.Ts - if err := this.writePacket(tcEvent.Len, this.ifIdex, time.Unix(0, int64(timeStamp)), tcEvent.Payload()); err != nil { - this.logger.Printf("%s\t save packet error %s .\n", this.Name(), err.Error()) - } - return -} - -// save pcapng file ,merge master key into pcapng file TODO -func (this *MOpenSSLProbe) savePcapng() error { - var i int = 0 - err := this.pcapWriter.WriteDecryptionSecretsBlock(pcapgo.DSB_SECRETS_TYPE_TLS, this.masterKeyBuffer.Bytes()) - if err != nil { - return err - } - this.tcPacketLocker.Lock() - defer this.tcPacketLocker.Unlock() - for _, packet := range this.tcPackets { - err := this.pcapWriter.WritePacket(packet.info, packet.data) - i++ - if err != nil { - return err - } - } - this.logger.Printf("%s\t save %d packets into pcapng file.\n", this.Name(), i) - if i == 0 { - this.logger.Printf("nothing captured, please check your network interface, see \"ecapture tls -h\" for more information.") - } - return this.pcapWriter.Flush() -} - -func (this *MOpenSSLProbe) createPcapng(netIfs []net.Interface) error { - pcapFile, err := os.OpenFile(this.pcapngFilename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return fmt.Errorf("error creating pcap file: %v", err) - } - - // TODO : write Application "ecapture.lua" to decode PID/Comm info. - pcapOption := pcapgo.NgWriterOptions{ - SectionInfo: pcapgo.NgSectionInfo{ - Hardware: "eCapture Hardware", - OS: "", - Application: "ecapture.lua", - Comment: "see https://ecapture.cc for more information. CFC4N ", - }, - } - // write interface description - ngIface := pcapgo.NgInterface{ - Name: this.conf.(*config.OpensslConfig).Ifname, - Comment: "eCapture (旁观者): github.com/gojue/ecapture", - Filter: "", - LinkType: layers.LinkTypeEthernet, - SnapLength: uint32(math.MaxUint16), - } - - pcapWriter, err := pcapgo.NewNgWriterInterface(pcapFile, ngIface, pcapOption) - if err != nil { - return err - } - - // insert other interfaces into pcapng file - for _, iface := range netIfs { - ngIface = pcapgo.NgInterface{ - Name: iface.Name, - Comment: "eCapture (旁观者): github.com/gojue/ecapture", - Filter: "", - LinkType: layers.LinkTypeEthernet, - SnapLength: uint32(math.MaxUint16), - } - - _, err := pcapWriter.AddInterface(ngIface) - if err != nil { - return err - } - } - - // Flush the header - err = pcapWriter.Flush() - if err != nil { - return err - } - - // TODO 保存数据包所属进程ID信息，以LRU Cache方式存储。 - this.pcapWriter = pcapWriter - return nil -} - -func (this *MOpenSSLProbe) writePacket(dataLen uint32, ifaceIdx int, timeStamp time.Time, packetBytes []byte) error { - info := gopacket.CaptureInfo{ - Timestamp: timeStamp, - CaptureLength: int(dataLen), - Length: int(dataLen), - InterfaceIndex: ifaceIdx, - } - - packet := &TcPacket{info: info, data: packetBytes} - - this.tcPackets = append(this.tcPackets, packet) - return nil -} - -func (this *MOpenSSLProbe) savePcapngSslKeyLog(sslKeyLog []byte) (err error) { - _, e := this.masterKeyBuffer.Write(sslKeyLog) - return e -} diff --git a/user/module/probe_tc.go b/user/module/probe_tc.go new file mode 100644 index 000000000..02872a46b --- /dev/null +++ b/user/module/probe_tc.go @@ -0,0 +1,147 @@ +package module + +import ( + "bytes" + "ecapture/user/event" + "fmt" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcapgo" + "math" + "net" + "os" + "sync" + "time" +) + +// packets of TC probe +type TcPacket struct { + info gopacket.CaptureInfo + data []byte +} + +type NetCaptureData struct { + PacketLength uint32 `json:"pktLen"` + ConfigIfaceIndex uint32 `json:"ifIndex"` +} + +func (NetCaptureData) GetSizeBytes() uint32 { + return 8 +} + +type MTCProbe struct { + //logger *log.Logger + //mName string + pcapngFilename string + ifIdex int + ifName string + pcapWriter *pcapgo.NgWriter + startTime uint64 + bootTime uint64 + tcPackets []*TcPacket + masterKeyBuffer *bytes.Buffer + tcPacketLocker *sync.Mutex +} + +func (this *MTCProbe) dumpTcSkb(tcEvent *event.TcSkbEvent) error { + var timeStamp = this.bootTime + tcEvent.Ts + return this.writePacket(tcEvent.Len, this.ifIdex, time.Unix(0, int64(timeStamp)), tcEvent.Payload()) +} + +// save pcapng file ,merge master key into pcapng file TODO +func (this *MTCProbe) savePcapng() (i int, err error) { + err = this.pcapWriter.WriteDecryptionSecretsBlock(pcapgo.DSB_SECRETS_TYPE_TLS, this.masterKeyBuffer.Bytes()) + if err != nil { + return + } + this.tcPacketLocker.Lock() + defer this.tcPacketLocker.Unlock() + for _, packet := range this.tcPackets { + err = this.pcapWriter.WritePacket(packet.info, packet.data) + i++ + if err != nil { + return + } + } + + if i == 0 { + return + } + err = this.pcapWriter.Flush() + return +} + +func (this *MTCProbe) createPcapng(netIfs []net.Interface) error { + pcapFile, err := os.OpenFile(this.pcapngFilename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return fmt.Errorf("error creating pcap file: %v", err) + } + + // TODO : write Application "ecapture.lua" to decode PID/Comm info. + pcapOption := pcapgo.NgWriterOptions{ + SectionInfo: pcapgo.NgSectionInfo{ + Hardware: "eCapture Hardware", + OS: "", + Application: "ecapture.lua", + Comment: "see https://ecapture.cc for more information. CFC4N ", + }, + } + // write interface description + ngIface := pcapgo.NgInterface{ + Name: this.ifName, + Comment: "eCapture (旁观者): github.com/gojue/ecapture", + Filter: "", + LinkType: layers.LinkTypeEthernet, + SnapLength: uint32(math.MaxUint16), + } + + pcapWriter, err := pcapgo.NewNgWriterInterface(pcapFile, ngIface, pcapOption) + if err != nil { + return err + } + + // insert other interfaces into pcapng file + for _, iface := range netIfs { + ngIface = pcapgo.NgInterface{ + Name: iface.Name, + Comment: "eCapture (旁观者): github.com/gojue/ecapture", + Filter: "", + LinkType: layers.LinkTypeEthernet, + SnapLength: uint32(math.MaxUint16), + } + + _, err := pcapWriter.AddInterface(ngIface) + if err != nil { + return err + } + } + + // Flush the header + err = pcapWriter.Flush() + if err != nil { + return err + } + + // TODO 保存数据包所属进程ID信息，以LRU Cache方式存储。 + this.pcapWriter = pcapWriter + return nil +} + +func (this *MTCProbe) writePacket(dataLen uint32, ifaceIdx int, timeStamp time.Time, packetBytes []byte) error { + info := gopacket.CaptureInfo{ + Timestamp: timeStamp, + CaptureLength: int(dataLen), + Length: int(dataLen), + InterfaceIndex: ifaceIdx, + } + + packet := &TcPacket{info: info, data: packetBytes} + + this.tcPackets = append(this.tcPackets, packet) + return nil +} + +func (this *MTCProbe) savePcapngSslKeyLog(sslKeyLog []byte) (err error) { + _, e := this.masterKeyBuffer.Write(sslKeyLog) + return e +} From 18680f9c76dbfa426b851fa057c1ff69c790e165 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Tue, 7 Mar 2023 23:26:59 +0800 Subject: [PATCH 3/8] kern: add gotls master secrets ebpf code. Signed-off-by: CFC4N --- kern/gotls_kern.c | 72 ++++++- user/event/event_mastersecret_gotls.go | 83 ++++++++ user/module/probe_gotls.go | 251 +++++++++++++++++++++---- user/module/probe_gotls_tc.go | 39 ++-- 4 files changed, 395 insertions(+), 50 deletions(-) create mode 100644 user/event/event_mastersecret_gotls.go diff --git a/kern/gotls_kern.c b/kern/gotls_kern.c index 340f8d733..ecb6f4805 100644 --- a/kern/gotls_kern.c +++ b/kern/gotls_kern.c @@ -17,6 +17,12 @@ #include "gotls.h" #include "tc.h" +#define GOTLS_RANDOM_SIZE 32 + +// max length is "CLIENT_HANDSHAKE_TRAFFIC_SECRET"=31 +#define MASTER_SECRET_KEY_LEN 32 +#define EVP_MAX_MD_SIZE 64 + struct go_tls_event { u64 ts_ns; u32 pid; @@ -26,6 +32,19 @@ struct go_tls_event { char data[MAX_DATA_SIZE_OPENSSL]; }; +struct mastersecret_gotls_t { + u8 lable[MASTER_SECRET_KEY_LEN]; + u8 client_random[GOTLS_RANDOM_SIZE]; + u8 secret_[EVP_MAX_MD_SIZE]; +}; + +/////////////////////////BPF MAPS //////////////////////////////// + +// bpf map +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); +} mastersecret_go_events SEC(".maps"); + struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); } events SEC(".maps"); @@ -106,4 +125,55 @@ int gotls_text_register(struct pt_regs *ctx) { SEC("uprobe/gotls_text_stack") int gotls_text_stack(struct pt_regs *ctx) { return gotls_text(ctx, false); -} \ No newline at end of file +} + + +/* +* crypto/tls/common.go +* func (c *Config) writeKeyLog(label string, clientRandom, secret []byte) error +*/ +static __always_inline int gotls_masterkey(struct pt_regs *ctx, bool is_register_abi) { + const char *label, *clientrandom, *secret; + void *lab_ptr, *cr_ptr, *secret_ptr; + lab_ptr = (void *)go_get_argument(ctx, is_register_abi, 2); + cr_ptr = (void *)go_get_argument(ctx, is_register_abi, 3); + secret_ptr = (void *)go_get_argument(ctx, is_register_abi, 4); + struct mastersecret_gotls_t mastersecret_gotls={}; + int ret = bpf_probe_read_user(&mastersecret_gotls.lable, sizeof(mastersecret_gotls.lable), (void*)lab_ptr); + if (ret < 0) { + debug_bpf_printk( + "gotls_masterkey read mastersecret lable failed, ret:%d, str:%d\n", ret, + str); + return 0; + } + + ret = bpf_probe_read_user(&mastersecret_gotls.client_random, sizeof(mastersecret_gotls.client_random), (void*)cr_ptr); + if (ret < 0) { + debug_bpf_printk( + "gotls_masterkey read mastersecret client_random failed, ret:%d, str:%d\n", ret, + str); + return 0; + } + + ret = bpf_probe_read_user(&mastersecret_gotls.secret_, sizeof(mastersecret_gotls.secret_), (void*)secret_ptr); + if (ret < 0) { + debug_bpf_printk( + "gotls_masterkey read mastersecret client_random failed, ret:%d, str:%d\n", ret, + str); + return 0; + } + + bpf_perf_event_output(ctx, &mastersecret_go_events, BPF_F_CURRENT_CPU, + &mastersecret_gotls, sizeof(struct mastersecret_gotls_t)); + return 0; +} + +SEC("uprobe/gotls_masterkey_register") +int gotls_masterkey_register(struct pt_regs *ctx) { + return gotls_masterkey(ctx, true); +} + +SEC("uprobe/gotls_masterkey_stack") +int gotls_masterkey_stack(struct pt_regs *ctx) { + return gotls_masterkey(ctx, false); +} diff --git a/user/event/event_mastersecret_gotls.go b/user/event/event_mastersecret_gotls.go new file mode 100644 index 000000000..6fe2f66c6 --- /dev/null +++ b/user/event/event_mastersecret_gotls.go @@ -0,0 +1,83 @@ +// Copyright 2022 CFC4N . All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package event + +import ( + "bytes" + "encoding/binary" + "fmt" +) + +// gotls_mastersecret_events + +const ( + GotlsRandomSize = 32 + MasterSecretKeyLen = 32 +) + +type MasterSecretGotlsEvent struct { + event_type EventType + Lable [MasterSecretKeyLen]byte `json:"lable"` // lable name + ClientRandom [GotlsRandomSize]byte `json:"clientRandom"` // Client Random + MasterSecret [EvpMaxMdSize]byte `json:"masterSecret"` // Master Secret + payload string +} + +func (this *MasterSecretGotlsEvent) Decode(payload []byte) (err error) { + buf := bytes.NewBuffer(payload) + if err = binary.Read(buf, binary.LittleEndian, &this.Lable); err != nil { + return + } + if err = binary.Read(buf, binary.LittleEndian, &this.ClientRandom); err != nil { + return + } + if err = binary.Read(buf, binary.LittleEndian, &this.MasterSecret); err != nil { + return + } + this.payload = fmt.Sprintf("%s %02x %02x", this.Lable, this.ClientRandom, this.MasterSecret) + return nil +} + +func (this *MasterSecretGotlsEvent) StringHex() string { + s := fmt.Sprintf("Lable:%s, ClientRandom:%02x", this.Lable, this.ClientRandom) + return s +} + +func (this *MasterSecretGotlsEvent) String() string { + s := fmt.Sprintf("Lable:%s, ClientRandom:%02x", this.Lable, this.ClientRandom) + return s +} + +func (this *MasterSecretGotlsEvent) Clone() IEventStruct { + event := new(MasterSecretGotlsEvent) + event.event_type = EventTypeModuleData + return event +} + +func (this *MasterSecretGotlsEvent) EventType() EventType { + return this.event_type +} + +func (this *MasterSecretGotlsEvent) GetUUID() string { + return fmt.Sprintf("%02X", this.ClientRandom) +} + +func (this *MasterSecretGotlsEvent) Payload() []byte { + return []byte(this.payload) +} + +func (this *MasterSecretGotlsEvent) PayloadLen() int { + return len(this.payload) +} diff --git a/user/module/probe_gotls.go b/user/module/probe_gotls.go index c2edfcd6c..3add621e2 100644 --- a/user/module/probe_gotls.go +++ b/user/module/probe_gotls.go @@ -8,8 +8,14 @@ import ( "ecapture/pkg/proc" "ecapture/user/config" "ecapture/user/event" + "errors" + "fmt" "log" "math" + "os" + "path/filepath" + "sync" + "time" "github.com/cilium/ebpf" manager "github.com/gojue/ebpfmanager" @@ -22,8 +28,8 @@ func init() { } const ( - goTlsHookFunc = "crypto/tls.(*Conn).writeRecordLocked" - goTlsMasterKeyFunc = "crypto/tls.(*Config).writeKeyLog" + goTlsHookFunc = "crypto/tls.(*Conn).writeRecordLocked" + goTlsMasterSecretFunc = "crypto/tls.(*Config).writeKeyLog" ) // GoTLSProbe represents a probe for Go SSL @@ -34,6 +40,11 @@ type GoTLSProbe struct { bpfManagerOptions manager.Options eventFuncMaps map[*ebpf.Map]event.IEventStruct eventMaps []*ebpf.Map + + keyloggerFilename string + keylogger *os.File + masterSecrets map[string]bool + eBPFProgramType EBPFPROGRAMTYPE path string isRegisterABI bool } @@ -45,6 +56,8 @@ func (this *GoTLSProbe) Init(ctx context.Context, l *log.Logger, cfg config.ICon this.eventMaps = make([]*ebpf.Map, 0, 2) this.eventFuncMaps = make(map[*ebpf.Map]event.IEventStruct) + + this.masterSecrets = make(map[string]bool) this.path = cfg.(*config.GoTLSConfig).Path ver, err := proc.ExtraceGoVersion(this.path) if err != nil { @@ -55,6 +68,41 @@ func (this *GoTLSProbe) Init(ctx context.Context, l *log.Logger, cfg config.ICon if ver.After(1, 17) { this.isRegisterABI = true } + + this.keyloggerFilename = "ecapture_masterkey.log" + file, err := os.OpenFile(this.keyloggerFilename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return err + } + this.keylogger = file + + var writeFile = this.conf.(*config.GoTLSConfig).Write + if len(writeFile) > 0 { + this.eBPFProgramType = EbpfprogramtypeOpensslTc + fileInfo, err := filepath.Abs(writeFile) + if err != nil { + return err + } + this.pcapngFilename = fileInfo + } else { + this.eBPFProgramType = EbpfprogramtypeOpensslUprobe + this.logger.Printf("%s\tmaster key keylogger: %s\n", this.Name(), this.keyloggerFilename) + } + + var ts unix.Timespec + err = unix.ClockGettime(unix.CLOCK_MONOTONIC, &ts) + if err != nil { + return err + } + startTime := ts.Nano() + bootTime := time.Now().UnixNano() - startTime + + this.startTime = uint64(startTime) + this.bootTime = uint64(bootTime) + + this.tcPackets = make([]*TcPacket, 0, 1024) + this.tcPacketLocker = &sync.Mutex{} + this.masterKeyBuffer = bytes.NewBuffer([]byte{}) return nil } @@ -63,17 +111,72 @@ func (this *GoTLSProbe) Name() string { } func (this *GoTLSProbe) Start() error { + return this.start() +} + +func (this *GoTLSProbe) start() error { + var err error + switch this.eBPFProgramType { + case EbpfprogramtypeOpensslTc: + this.logger.Printf("%s\tTC MODEL\n", this.Name()) + err = this.setupManagersTC() + case EbpfprogramtypeOpensslUprobe: + this.logger.Printf("%s\tUPROBE MODEL\n", this.Name()) + err = this.setupManagersUprobe() + default: + this.logger.Printf("%s\tUPROBE MODEL\n", this.Name()) + err = this.setupManagersUprobe() + } + if err != nil { + return err + } + + var bpfFileName = this.geteBPFName("user/bytecode/gotls_kern.o") + this.logger.Printf("%s\tBPF bytecode filename:%s\n", this.Name(), bpfFileName) + byteBuf, err := assets.Asset(bpfFileName) + if err != nil { + return err + } + + if err = this.bpfManager.InitWithOptions(bytes.NewReader(byteBuf), this.bpfManagerOptions); err != nil { + return fmt.Errorf("couldn't init manager %v", err) + } + // start the bootstrap manager + if err = this.bpfManager.Start(); err != nil { + return fmt.Errorf("couldn't start bootstrap manager %v .", err) + } + + // 加载map信息，map对应events decode表。 + switch this.eBPFProgramType { + case EbpfprogramtypeOpensslTc: + err = this.initDecodeFunTC() + case EbpfprogramtypeOpensslUprobe: + err = this.initDecodeFun() + default: + err = this.initDecodeFun() + } + if err != nil { + return err + } + return nil +} + +func (this *GoTLSProbe) setupManagersUprobe() error { var ( - sec string - fn string + sec, ms_sec string + fn, ms_fn string ) if this.isRegisterABI { sec = "uprobe/gotls_text_register" fn = "gotls_text_register" + ms_sec = "uprobe/gotls_masterkey_register" + ms_fn = "gotls_masterkey_register" } else { sec = "uprobe/gotls_text_stack" fn = "gotls_text_stack" + ms_sec = "uprobe/gotls_masterkey_stack" + ms_fn = "gotls_masterkey_stack" } this.logger.Printf("%s\teBPF Function Name:%s, isRegisterABI:%t\n", this.Name(), fn, this.isRegisterABI) this.bpfManager = &manager.Manager{ @@ -84,46 +187,51 @@ func (this *GoTLSProbe) Start() error { AttachToFuncName: goTlsHookFunc, BinaryPath: this.path, }, - - //{ - // Section: "uprobe/gotls_masterkey", - // EbpfFuncName: "gotls_masterkey", - // AttachToFuncName: "crypto/tls.(*Conn).writeRecordLocked", - // BinaryPath: this.path, - //}, + // gotls master secrets + // crypto/tls.(*Config).writeKeyLog + // crypto/tls/common.go + /* + func (c *Config) writeKeyLog(label string, clientRandom, secret []byte) error { + } + */ + { + Section: ms_sec, + EbpfFuncName: ms_fn, + AttachToFuncName: goTlsMasterSecretFunc, + BinaryPath: this.path, + UID: "uprobe_gotls_master_secret", + }, }, Maps: []*manager.Map{ + { + Name: "mastersecret_go_events", + }, { Name: "events", }, }, } - // crypto/tls.(*Config).writeKeyLog - // crypto/tls/common.go - /* - func (c *Config) writeKeyLog(label string, clientRandom, secret []byte) error { - } - */ + this.bpfManagerOptions = manager.Options{ + DefaultKProbeMaxActive: 512, - var bpfFileName = this.geteBPFName("user/bytecode/gotls_kern.o") - this.logger.Printf("%s\tBPF bytecode filename:%s\n", this.Name(), bpfFileName) - byteBuf, err := assets.Asset(bpfFileName) - if err != nil { - return err - } + VerifierOptions: ebpf.CollectionOptions{ + Programs: ebpf.ProgramOptions{ + LogSize: 2097152, + }, + }, - opts := manager.Options{ RLimit: &unix.Rlimit{ Cur: math.MaxUint64, Max: math.MaxUint64, }, } - if err := this.bpfManager.InitWithOptions(bytes.NewReader(byteBuf), opts); err != nil { - return err - } - return this.bpfManager.Start() + if this.conf.EnableGlobalVar() { + // 填充 RewriteContants 对应map + this.bpfManagerOptions.ConstantEditors = this.constantEditor() + } + return nil } // 通过elf的常量替换方式传递数据 @@ -140,7 +248,7 @@ func (this *GoTLSProbe) constantEditor() []manager.ConstantEditor { }, { Name: "target_port", - Value: uint64(this.conf.(*config.OpensslConfig).Port), + Value: uint64(this.conf.(*config.GoTLSConfig).Port), }, } @@ -159,22 +267,91 @@ func (this *GoTLSProbe) constantEditor() []manager.ConstantEditor { return editor } -func (this *GoTLSProbe) Events() []*ebpf.Map { - var maps []*ebpf.Map +func (this *GoTLSProbe) initDecodeFun() error { + + m, found, err := this.bpfManager.GetMap("events") + if err != nil { + return err + } + if !found { + return errors.New("cant found map:tls_events") + } - m, ok, err := this.bpfManager.GetMap("events") - if err != nil || !ok { - return maps + this.eventMaps = append(this.eventMaps, m) + gotlsEvent := &event.GoTLSEvent{} + //sslEvent.SetModule(this) + this.eventFuncMaps[m] = gotlsEvent + // master secrets map at ebpf code + MasterkeyEventsMap, found, err := this.bpfManager.GetMap("mastersecret_go_events") + if err != nil { + return err + } + if !found { + return errors.New("cant found map:mastersecret_events") } + this.eventMaps = append(this.eventMaps, MasterkeyEventsMap) + + var masterkeyEvent event.IEventStruct - maps = append(maps, m) - return maps + // goTLS Event struct + masterkeyEvent = &event.MasterSecretGotlsEvent{} + + this.eventFuncMaps[MasterkeyEventsMap] = masterkeyEvent + return nil } func (this *GoTLSProbe) DecodeFun(m *ebpf.Map) (event.IEventStruct, bool) { - return &event.GoTLSEvent{}, true + fun, found := this.eventFuncMaps[m] + return fun, found } func (this *GoTLSProbe) Close() error { return this.Module.Close() } + +func (this *GoTLSProbe) saveMasterSecret(secretEvent *event.MasterSecretGotlsEvent) { + var k = fmt.Sprintf("%s-%02x", secretEvent.Lable, secretEvent.ClientRandom) + + _, f := this.masterSecrets[k] + if f { + // 已存在该随机数的masterSecret，不需要重复写入 + return + } + + // TODO 保存多个lable 整组里？？？ + // save to file + //var b *bytes.Buffer + //l, e := this.keylogger.WriteString(b.String()) + //if e != nil { + // this.logger.Fatalf("%s: save masterSecrets to file error:%s", secretEvent.String(), e.Error()) + // return + //} + + // + this.logger.Printf("%s: save masterSecrets %02x to file success, %d bytes", secretEvent.String(), secretEvent.ClientRandom, len(secretEvent.String())) + /* + switch this.eBPFProgramType { + case EbpfprogramtypeOpensslTc: + e = this.savePcapngSslKeyLog(b.Bytes()) + if e != nil { + this.logger.Fatalf("%s: save masterSecrets to pcapng error:%s", secretEvent.String(), e.Error()) + return + } + default: + } + */ +} + +func (this *GoTLSProbe) Dispatcher(eventStruct event.IEventStruct) { + // detect eventStruct type + switch eventStruct.(type) { + case *event.MasterSecretGotlsEvent: + this.saveMasterSecret(eventStruct.(*event.MasterSecretGotlsEvent)) + case *event.TcSkbEvent: + err := this.dumpTcSkb(eventStruct.(*event.TcSkbEvent)) + if err != nil { + this.logger.Printf("%s\t save packet error %s .\n", this.Name(), err.Error()) + } + } + //this.logger.Println(eventStruct) +} diff --git a/user/module/probe_gotls_tc.go b/user/module/probe_gotls_tc.go index 413afaf29..a5521eee3 100644 --- a/user/module/probe_gotls_tc.go +++ b/user/module/probe_gotls_tc.go @@ -46,7 +46,7 @@ func (this *GoTLSProbe) setupManagersTC() error { this.logger.Printf("%s\tHOOK type:golang elf, binrayPath:%s\n", this.Name(), this.path) this.logger.Printf("%s\tIfname:%s, Ifindex:%d, Port:%d, Pcapng filepath:%s\n", this.Name(), this.ifName, this.ifIdex, this.conf.(*config.GoTLSConfig).Port, this.pcapngFilename) - this.logger.Printf("%s\tHook masterKey function:%s\n", this.Name(), goTlsMasterKeyFunc) + this.logger.Printf("%s\tHook masterKey function:%s\n", this.Name(), goTlsMasterSecretFunc) // create pcapng writer netIfs, err := net.Interfaces() @@ -59,6 +59,19 @@ func (this *GoTLSProbe) setupManagersTC() error { return err } + var ( + sec string + fn string + ) + + if this.isRegisterABI { + sec = "uprobe/gotls_masterkey_register" + fn = "gotls_masterkey_register" + } else { + sec = "uprobe/gotls_masterkey_stack" + fn = "gotls_masterkey_stack" + } + this.bpfManager = &manager.Manager{ Probes: []*manager.Probe{ { @@ -77,18 +90,17 @@ func (this *GoTLSProbe) setupManagersTC() error { // gotls master secrets { - // TODO - Section: "uprobe/gotls_writekeylog", - EbpfFuncName: "probe_gotls_writekeylog", - AttachToFuncName: goTlsMasterKeyFunc, + Section: sec, + EbpfFuncName: fn, + AttachToFuncName: goTlsMasterSecretFunc, BinaryPath: this.path, - UID: "uprobe_gotls_master_key", + UID: "uprobe_gotls_master_secret", }, }, Maps: []*manager.Map{ { - Name: "mastersecret_events", + Name: "mastersecret_go_events", }, { Name: "skb_events", @@ -132,8 +144,8 @@ func (this *GoTLSProbe) initDecodeFunTC() error { //sslEvent.SetModule(this) this.eventFuncMaps[SkbEventsMap] = sslEvent - // TODO write a master secrets map at ebpf code - MasterkeyEventsMap, found, err := this.bpfManager.GetMap("mastersecret_events") + // master secrets map at ebpf code + MasterkeyEventsMap, found, err := this.bpfManager.GetMap("mastersecret_go_events") if err != nil { return err } @@ -144,10 +156,13 @@ func (this *GoTLSProbe) initDecodeFunTC() error { var masterkeyEvent event.IEventStruct - // TODO goTLS Event struct - masterkeyEvent = &event.MasterSecretEvent{} + // goTLS Event struct + masterkeyEvent = &event.MasterSecretGotlsEvent{} - //masterkeyEvent.SetModule(this) this.eventFuncMaps[MasterkeyEventsMap] = masterkeyEvent return nil } + +func (this *GoTLSProbe) Events() []*ebpf.Map { + return this.eventMaps +} From 56da6cc1241e290a12fca5975ced9af30362196c Mon Sep 17 00:00:00 2001 From: CFC4N Date: Thu, 9 Mar 2023 23:41:25 +0800 Subject: [PATCH 4/8] kern: rename gotls.h to go_argument.h Signed-off-by: CFC4N --- kern/{gotls.h => go_argument.h} | 0 kern/gotls_kern.c | 135 +++++++++++++++++++++----------- 2 files changed, 91 insertions(+), 44 deletions(-) rename kern/{gotls.h => go_argument.h} (100%) diff --git a/kern/gotls.h b/kern/go_argument.h similarity index 100% rename from kern/gotls.h rename to kern/go_argument.h diff --git a/kern/gotls_kern.c b/kern/gotls_kern.c index ecb6f4805..60e36e76e 100644 --- a/kern/gotls_kern.c +++ b/kern/gotls_kern.c @@ -14,7 +14,7 @@ /* Copyright © 2022 Hengqi Chen */ #include "ecapture.h" -#include "gotls.h" +#include "go_argument.h" #include "tc.h" #define GOTLS_RANDOM_SIZE 32 @@ -33,9 +33,12 @@ struct go_tls_event { }; struct mastersecret_gotls_t { - u8 lable[MASTER_SECRET_KEY_LEN]; + u8 label[MASTER_SECRET_KEY_LEN]; + u8 labellen; u8 client_random[GOTLS_RANDOM_SIZE]; + u8 client_random_len; u8 secret_[EVP_MAX_MD_SIZE]; + u8 secret_len; }; /////////////////////////BPF MAPS //////////////////////////////// @@ -77,13 +80,15 @@ static __always_inline struct go_tls_event *get_gotls_event() { return bpf_map_lookup_elem(>e_context, &id); } -static __always_inline int gotls_text(struct pt_regs *ctx, bool is_register_abi) { +static __always_inline int gotls_text(struct pt_regs *ctx, + bool is_register_abi) { s32 record_type, len; const char *str; - void * record_type_ptr; - void * len_ptr; + void *record_type_ptr; + void *len_ptr; record_type_ptr = (void *)go_get_argument(ctx, is_register_abi, 2); - bpf_probe_read_kernel(&record_type, sizeof(record_type), (void *)&record_type_ptr); + bpf_probe_read_kernel(&record_type, sizeof(record_type), + (void *)&record_type_ptr); str = (void *)go_get_argument(ctx, is_register_abi, 3); len_ptr = (void *)go_get_argument(ctx, is_register_abi, 4); bpf_probe_read_kernel(&len, sizeof(len), (void *)&len_ptr); @@ -99,7 +104,8 @@ static __always_inline int gotls_text(struct pt_regs *ctx, bool is_register_abi) } event->data_len = len; - int ret = bpf_probe_read_user(&event->data, sizeof(event->data), (void*)str); + int ret = + bpf_probe_read_user(&event->data, sizeof(event->data), (void *)str); if (ret < 0) { debug_bpf_printk( "gotls_text bpf_probe_read_user_str failed, ret:%d, str:%d\n", ret, @@ -111,69 +117,110 @@ static __always_inline int gotls_text(struct pt_regs *ctx, bool is_register_abi) return 0; } -// capture golang tls plaintext, supported golang stack-based ABI (go version >= 1.17) -// type recordType uint8 -// writeRecordLocked(typ recordType, data []byte) +// capture golang tls plaintext, supported golang stack-based ABI (go version +// >= 1.17) type recordType uint8 writeRecordLocked(typ recordType, data []byte) SEC("uprobe/gotls_text_register") -int gotls_text_register(struct pt_regs *ctx) { - return gotls_text(ctx, true); -} +int gotls_text_register(struct pt_regs *ctx) { return gotls_text(ctx, true); } -// capture golang tls plaintext, supported golang stack-based ABI (go version < 1.17) -// type recordType uint8 -// writeRecordLocked(typ recordType, data []byte) +// capture golang tls plaintext, supported golang stack-based ABI (go version +// < 1.17) type recordType uint8 writeRecordLocked(typ recordType, data []byte) SEC("uprobe/gotls_text_stack") -int gotls_text_stack(struct pt_regs *ctx) { - return gotls_text(ctx, false); -} - +int gotls_text_stack(struct pt_regs *ctx) { return gotls_text(ctx, false); } /* -* crypto/tls/common.go -* func (c *Config) writeKeyLog(label string, clientRandom, secret []byte) error -*/ -static __always_inline int gotls_masterkey(struct pt_regs *ctx, bool is_register_abi) { - const char *label, *clientrandom, *secret; + * crypto/tls/common.go + * func (c *Config) writeKeyLog(label string, clientRandom, secret []byte) error + */ +static __always_inline int gotls_mastersecret(struct pt_regs *ctx, + bool is_register_abi) { + // const char *label, *clientrandom, *secret; void *lab_ptr, *cr_ptr, *secret_ptr; + void *lab_len_ptr, *cr_len_ptr, *secret_len_ptr; + s32 lab_len, cr_len, secret_len; + + /* + * + * in golang struct, slice header like this + * type slice struct { + * array unsafe.Pointer + * len int + * cap int + * } + * so, arument index are in the order one by one + * + */ lab_ptr = (void *)go_get_argument(ctx, is_register_abi, 2); - cr_ptr = (void *)go_get_argument(ctx, is_register_abi, 3); - secret_ptr = (void *)go_get_argument(ctx, is_register_abi, 4); - struct mastersecret_gotls_t mastersecret_gotls={}; - int ret = bpf_probe_read_user(&mastersecret_gotls.lable, sizeof(mastersecret_gotls.lable), (void*)lab_ptr); + lab_len_ptr = (void *)go_get_argument(ctx, is_register_abi, 3); + cr_ptr = (void *)go_get_argument(ctx, is_register_abi, 4); + cr_len_ptr = (void *)go_get_argument(ctx, is_register_abi, 5); + secret_ptr = (void *)go_get_argument(ctx, is_register_abi, 7); + secret_len_ptr = (void *)go_get_argument(ctx, is_register_abi, 8); + + bpf_probe_read_kernel(&lab_len, sizeof(lab_len), (void *)&lab_len_ptr); + bpf_probe_read_kernel(&cr_len, sizeof(lab_len), (void *)&cr_len_ptr); + bpf_probe_read_kernel(&secret_len, sizeof(lab_len), + (void *)&secret_len_ptr); + + debug_bpf_printk( + "gotls_mastersecret read params length failed, lab_len:%d, cr_len:%d, " + "secret_len:%d\n", + lab_len, cr_len, secret_len); + + if (lab_len <= 0 || cr_len <= 0 || secret_len <= 0) { + return 0; + } + + struct mastersecret_gotls_t mastersecret_gotls = {}; + mastersecret_gotls.labellen = lab_len; + mastersecret_gotls.client_random_len = cr_len; + mastersecret_gotls.secret_len = secret_len; + int ret = bpf_probe_read_user_str(&mastersecret_gotls.label, + sizeof(mastersecret_gotls.label), + (void *)lab_ptr); if (ret < 0) { debug_bpf_printk( - "gotls_masterkey read mastersecret lable failed, ret:%d, str:%d\n", ret, - str); + "gotls_mastersecret read mastersecret label failed, ret:%d, " + "lab_ptr:%p\n", + ret, lab_ptr); return 0; } - ret = bpf_probe_read_user(&mastersecret_gotls.client_random, sizeof(mastersecret_gotls.client_random), (void*)cr_ptr); + debug_bpf_printk("gotls_mastersecret read mastersecret label%s\n", + mastersecret_gotls.label); + ret = bpf_probe_read_user_str(&mastersecret_gotls.client_random, + sizeof(mastersecret_gotls.client_random), + (void *)cr_ptr); if (ret < 0) { debug_bpf_printk( - "gotls_masterkey read mastersecret client_random failed, ret:%d, str:%d\n", ret, - str); + "gotls_mastersecret read mastersecret client_random failed, " + "ret:%d, cr_ptr:%p\n", + ret, cr_ptr); return 0; } - ret = bpf_probe_read_user(&mastersecret_gotls.secret_, sizeof(mastersecret_gotls.secret_), (void*)secret_ptr); + ret = bpf_probe_read_user_str(&mastersecret_gotls.secret_, + sizeof(mastersecret_gotls.secret_), + (void *)secret_ptr); if (ret < 0) { debug_bpf_printk( - "gotls_masterkey read mastersecret client_random failed, ret:%d, str:%d\n", ret, - str); + "gotls_mastersecret read mastersecret secret_ failed, ret:%d, " + "secret_ptr:%p\n", + ret, secret_ptr); return 0; } bpf_perf_event_output(ctx, &mastersecret_go_events, BPF_F_CURRENT_CPU, - &mastersecret_gotls, sizeof(struct mastersecret_gotls_t)); + &mastersecret_gotls, + sizeof(struct mastersecret_gotls_t)); return 0; } -SEC("uprobe/gotls_masterkey_register") -int gotls_masterkey_register(struct pt_regs *ctx) { - return gotls_masterkey(ctx, true); +SEC("uprobe/gotls_mastersecret_register") +int gotls_mastersecret_register(struct pt_regs *ctx) { + return gotls_mastersecret(ctx, true); } -SEC("uprobe/gotls_masterkey_stack") -int gotls_masterkey_stack(struct pt_regs *ctx) { - return gotls_masterkey(ctx, false); +SEC("uprobe/gotls_mastersecret_stack") +int gotls_mastersecret_stack(struct pt_regs *ctx) { + return gotls_mastersecret(ctx, false); } From 575e086aeed61f21687867de28a4785e7059dcb1 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Thu, 9 Mar 2023 23:53:52 +0800 Subject: [PATCH 5/8] kern: adjust prompt message Signed-off-by: CFC4N --- kern/gotls_kern.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kern/gotls_kern.c b/kern/gotls_kern.c index 60e36e76e..82945799f 100644 --- a/kern/gotls_kern.c +++ b/kern/gotls_kern.c @@ -35,7 +35,7 @@ struct go_tls_event { struct mastersecret_gotls_t { u8 label[MASTER_SECRET_KEY_LEN]; u8 labellen; - u8 client_random[GOTLS_RANDOM_SIZE]; + u8 client_random[EVP_MAX_MD_SIZE]; u8 client_random_len; u8 secret_[EVP_MAX_MD_SIZE]; u8 secret_len; @@ -161,15 +161,15 @@ static __always_inline int gotls_mastersecret(struct pt_regs *ctx, bpf_probe_read_kernel(&secret_len, sizeof(lab_len), (void *)&secret_len_ptr); - debug_bpf_printk( - "gotls_mastersecret read params length failed, lab_len:%d, cr_len:%d, " - "secret_len:%d\n", - lab_len, cr_len, secret_len); - if (lab_len <= 0 || cr_len <= 0 || secret_len <= 0) { return 0; } + debug_bpf_printk( + "gotls_mastersecret read params length success, lab_len:%d, cr_len:%d, " + "secret_len:%d\n", + lab_len, cr_len, secret_len); + struct mastersecret_gotls_t mastersecret_gotls = {}; mastersecret_gotls.labellen = lab_len; mastersecret_gotls.client_random_len = cr_len; From 8712a317b0ddc33d2545aead2e540084c450c762 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Thu, 9 Mar 2023 23:59:05 +0800 Subject: [PATCH 6/8] user: improve gotls key capture module write to ecapture_masterkey.log file, and save network traffic to pcapng file. Signed-off-by: CFC4N --- user/event/event_mastersecret_gotls.go | 39 +++++++++++---- user/module/probe_gotls.go | 67 ++++++++++++++++---------- user/module/probe_gotls_tc.go | 8 +-- 3 files changed, 76 insertions(+), 38 deletions(-) diff --git a/user/event/event_mastersecret_gotls.go b/user/event/event_mastersecret_gotls.go index 6fe2f66c6..59bc3dab2 100644 --- a/user/event/event_mastersecret_gotls.go +++ b/user/event/event_mastersecret_gotls.go @@ -28,35 +28,56 @@ const ( ) type MasterSecretGotlsEvent struct { - event_type EventType - Lable [MasterSecretKeyLen]byte `json:"lable"` // lable name - ClientRandom [GotlsRandomSize]byte `json:"clientRandom"` // Client Random - MasterSecret [EvpMaxMdSize]byte `json:"masterSecret"` // Master Secret - payload string + event_type EventType + Label [MasterSecretKeyLen]byte `json:"label"` // label name + LabelLen uint8 `json:"labelLen"` + ClientRandom [EvpMaxMdSize]byte `json:"clientRandom"` // Client Random + ClientRandomLen uint8 `json:"clientRandomLen"` + MasterSecret [EvpMaxMdSize]byte `json:"masterSecret"` // Master Secret + MasterSecretLen uint8 `json:"masterSecretLen"` + payload string } func (this *MasterSecretGotlsEvent) Decode(payload []byte) (err error) { buf := bytes.NewBuffer(payload) - if err = binary.Read(buf, binary.LittleEndian, &this.Lable); err != nil { + if err = binary.Read(buf, binary.LittleEndian, &this.Label); err != nil { + return + } + if err = binary.Read(buf, binary.LittleEndian, &this.LabelLen); err != nil { return } if err = binary.Read(buf, binary.LittleEndian, &this.ClientRandom); err != nil { return } + if err = binary.Read(buf, binary.LittleEndian, &this.ClientRandomLen); err != nil { + return + } if err = binary.Read(buf, binary.LittleEndian, &this.MasterSecret); err != nil { return } - this.payload = fmt.Sprintf("%s %02x %02x", this.Lable, this.ClientRandom, this.MasterSecret) + if err = binary.Read(buf, binary.LittleEndian, &this.MasterSecretLen); err != nil { + return + } + if int(this.LabelLen) > len(this.Label) { + return fmt.Errorf("invalid label length, LablenLen:%d, len(Label):%d", this.LabelLen, len(this.Label)) + } + if int(this.ClientRandomLen) > len(this.ClientRandom) { + return fmt.Errorf("invalid label length, ClientRandomLen:%d, len(ClientRandom):%d", this.ClientRandomLen, len(this.ClientRandom)) + } + if int(this.MasterSecretLen) > len(this.MasterSecret) { + return fmt.Errorf("invalid label length, MasterSecretLen:%d, len(MasterSecret):%d", this.MasterSecretLen, len(this.MasterSecret)) + } + this.payload = fmt.Sprintf("%s %02x %02x", this.Label, this.ClientRandom, this.MasterSecret) return nil } func (this *MasterSecretGotlsEvent) StringHex() string { - s := fmt.Sprintf("Lable:%s, ClientRandom:%02x", this.Lable, this.ClientRandom) + s := fmt.Sprintf("Label%s, ClientRandom:%02x, secret:%02x", this.Label[0:this.LabelLen], this.ClientRandom[0:this.ClientRandomLen], this.MasterSecret[0:this.MasterSecretLen]) return s } func (this *MasterSecretGotlsEvent) String() string { - s := fmt.Sprintf("Lable:%s, ClientRandom:%02x", this.Lable, this.ClientRandom) + s := fmt.Sprintf("Label:%s, ClientRandom:%02x, secret:%02x", this.Label[0:this.LabelLen], this.ClientRandom[0:this.ClientRandomLen], this.MasterSecret[0:this.MasterSecretLen]) return s } diff --git a/user/module/probe_gotls.go b/user/module/probe_gotls.go index 3add621e2..b76b56bca 100644 --- a/user/module/probe_gotls.go +++ b/user/module/probe_gotls.go @@ -170,13 +170,13 @@ func (this *GoTLSProbe) setupManagersUprobe() error { if this.isRegisterABI { sec = "uprobe/gotls_text_register" fn = "gotls_text_register" - ms_sec = "uprobe/gotls_masterkey_register" - ms_fn = "gotls_masterkey_register" + ms_sec = "uprobe/gotls_mastersecret_register" + ms_fn = "gotls_mastersecret_register" } else { sec = "uprobe/gotls_text_stack" fn = "gotls_text_stack" - ms_sec = "uprobe/gotls_masterkey_stack" - ms_fn = "gotls_masterkey_stack" + ms_sec = "uprobe/gotls_mastersecret_stack" + ms_fn = "gotls_mastersecret_stack" } this.logger.Printf("%s\teBPF Function Name:%s, isRegisterABI:%t\n", this.Name(), fn, this.isRegisterABI) this.bpfManager = &manager.Manager{ @@ -306,11 +306,34 @@ func (this *GoTLSProbe) DecodeFun(m *ebpf.Map) (event.IEventStruct, bool) { } func (this *GoTLSProbe) Close() error { + + if this.eBPFProgramType == EbpfprogramtypeOpensslTc { + this.logger.Printf("%s\tsaving pcapng file %s\n", this.Name(), this.pcapngFilename) + i, err := this.savePcapng() + if err != nil { + this.logger.Printf("%s\tsave pcanNP failed, error:%v. \n", this.Name(), err) + } + if i == 0 { + this.logger.Printf("nothing captured, please check your network interface, see \"ecapture tls -h\" for more information.") + } else { + this.logger.Printf("%s\t save %d packets into pcapng file.\n", this.Name(), i) + } + } + + this.logger.Printf("%s\tclose. \n", this.Name()) + if err := this.bpfManager.Stop(manager.CleanAll); err != nil { + return fmt.Errorf("couldn't stop manager %v .", err) + } return this.Module.Close() } func (this *GoTLSProbe) saveMasterSecret(secretEvent *event.MasterSecretGotlsEvent) { - var k = fmt.Sprintf("%s-%02x", secretEvent.Lable, secretEvent.ClientRandom) + var label, clientRandom, secret string + label = string(secretEvent.Label[0:secretEvent.LabelLen]) + clientRandom = string(secretEvent.ClientRandom[0:secretEvent.ClientRandomLen]) + secret = string(secretEvent.MasterSecret[0:secretEvent.MasterSecretLen]) + + var k = fmt.Sprintf("%s-%02x", label, clientRandom) _, f := this.masterSecrets[k] if f { @@ -320,26 +343,20 @@ func (this *GoTLSProbe) saveMasterSecret(secretEvent *event.MasterSecretGotlsEve // TODO 保存多个lable 整组里？？？ // save to file - //var b *bytes.Buffer - //l, e := this.keylogger.WriteString(b.String()) - //if e != nil { - // this.logger.Fatalf("%s: save masterSecrets to file error:%s", secretEvent.String(), e.Error()) - // return - //} - - // - this.logger.Printf("%s: save masterSecrets %02x to file success, %d bytes", secretEvent.String(), secretEvent.ClientRandom, len(secretEvent.String())) - /* - switch this.eBPFProgramType { - case EbpfprogramtypeOpensslTc: - e = this.savePcapngSslKeyLog(b.Bytes()) - if e != nil { - this.logger.Fatalf("%s: save masterSecrets to pcapng error:%s", secretEvent.String(), e.Error()) - return - } - default: - } - */ + var b string + b = fmt.Sprintf("%s %02x %02x\n", label, clientRandom, secret) + l, e := this.keylogger.WriteString(b) + if e != nil { + this.logger.Fatalf("%s: save masterSecrets to file error:%s", secretEvent.String(), e.Error()) + return + } + this.logger.Printf("%s: save CLIENT_RANDOM %02x to file success, %d bytes", label, clientRandom, l) + e = this.savePcapngSslKeyLog([]byte(b)) + if e != nil { + this.logger.Fatalf("%s: save masterSecrets to pcapng error:%s", secretEvent.String(), e.Error()) + return + } + } func (this *GoTLSProbe) Dispatcher(eventStruct event.IEventStruct) { diff --git a/user/module/probe_gotls_tc.go b/user/module/probe_gotls_tc.go index a5521eee3..fdea94132 100644 --- a/user/module/probe_gotls_tc.go +++ b/user/module/probe_gotls_tc.go @@ -65,11 +65,11 @@ func (this *GoTLSProbe) setupManagersTC() error { ) if this.isRegisterABI { - sec = "uprobe/gotls_masterkey_register" - fn = "gotls_masterkey_register" + sec = "uprobe/gotls_mastersecret_register" + fn = "gotls_mastersecret_register" } else { - sec = "uprobe/gotls_masterkey_stack" - fn = "gotls_masterkey_stack" + sec = "uprobe/gotls_mastersecret_stack" + fn = "gotls_mastersecret_stack" } this.bpfManager = &manager.Manager{ From 4931346bd5e5312b8ad126ade9036e772f03e914 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Fri, 10 Mar 2023 00:03:50 +0800 Subject: [PATCH 7/8] kern: make format Signed-off-by: CFC4N --- kern/boringssl_masterkey.h | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/kern/boringssl_masterkey.h b/kern/boringssl_masterkey.h index 515ca0117..3c24c8852 100644 --- a/kern/boringssl_masterkey.h +++ b/kern/boringssl_masterkey.h @@ -113,7 +113,8 @@ static __always_inline struct mastersecret_bssl_t *make_event() { // in boringssl, the master secret is stored in src/ssl/handshake.cc 581 // const SSL_SESSION *ssl_handshake_session(const SSL_HANDSHAKE *hs) { -static __always_inline u64 get_session_addr(void *ssl_st_ptr, u64 s3_address, u64 ssl_hs_st_ptr) { +static __always_inline u64 get_session_addr(void *ssl_st_ptr, u64 s3_address, + u64 ssl_hs_st_ptr) { u64 tmp_address; int ret; @@ -220,7 +221,8 @@ int probe_ssl_master_key(struct pt_regs *ctx) { ret = bpf_probe_read_user(&ssl_hs_st_addr, sizeof(ssl_hs_st_addr), ssl_hs_st_ptr); if (ret || ssl_hs_st_addr == 0) { -// debug_bpf_printk("bpf_probe_read ssl_hs_st_ptr failed, ret :%d\n", ret); + // debug_bpf_printk("bpf_probe_read ssl_hs_st_ptr failed, ret + // :%d\n", ret); return 0; } @@ -241,7 +243,7 @@ int probe_ssl_master_key(struct pt_regs *ctx) { (u64 *)(ssl_hs_st_addr + BSSL__SSL_HANDSHAKE_CLIENT_VERSION); ret = bpf_probe_read_user(&client_version, sizeof(client_version), ssl_hs_cv_ptr); -// if (ret || client_version == 0) { + // if (ret || client_version == 0) { if (ret) { debug_bpf_printk( "bpf_probe_read ssl_hs_st_ptr failed, ret :%d, client_version:%d\n", @@ -261,15 +263,17 @@ int probe_ssl_master_key(struct pt_regs *ctx) { // ssl_client_hs_state_t::ssl3_hs_state=5 // tls13_server_hs_state_t::state13_read_second_client_flight -// if (ssl3_hs_state.state == 5 && ssl3_hs_state.tls13_state < 8) { -// return 0; -// } + // if (ssl3_hs_state.state == 5 && ssl3_hs_state.tls13_state < 8) { + // return 0; + // } ///////////// debug info ///////// - debug_bpf_printk("client_version:%d, state:%d, tls13_state:%d\n", client_version, ssl3_hs_state.state, + debug_bpf_printk("client_version:%d, state:%d, tls13_state:%d\n", + client_version, ssl3_hs_state.state, ssl3_hs_state.tls13_state); // debug_bpf_printk("openssl uprobe/SSL_write masterKey PID :%d\n", pid); - debug_bpf_printk("TLS version :%d, hash_len:%d, \n", mastersecret->version, hash_len); + debug_bpf_printk("TLS version :%d, hash_len:%d, \n", mastersecret->version, + hash_len); // 判断当前tls链接状态 // handshake->handshake_finalized = hs_st_addr + BSSL__SSL_HANDSHAKE_HINTS + s32 all_bool; @@ -289,15 +293,16 @@ int probe_ssl_master_key(struct pt_regs *ctx) { if (mastersecret->version != TLS1_3_VERSION) { // state12_finish_server_handshake // state12_done - if (ssl3_hs_state.state < 20) { + if (ssl3_hs_state.state < 20) { // not finished yet. return 0; } // Get ssl_session_st pointer u64 ssl_session_st_addr; - ssl_session_st_addr = get_session_addr(ssl_st_ptr, s3_address, ssl_hs_st_addr); + ssl_session_st_addr = + get_session_addr(ssl_st_ptr, s3_address, ssl_hs_st_addr); if (ssl_session_st_addr == 0) { -// debug_bpf_printk("ssl_session_st_addr is null\n"); + // debug_bpf_printk("ssl_session_st_addr is null\n"); return 0; } debug_bpf_printk("s3_address:%llx, ssl_session_st_addr addr :%llx\n", From 6a6a82dda0b6b260f817a14dcdb20d673e552335 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Fri, 10 Mar 2023 21:55:11 +0800 Subject: [PATCH 8/8] chore. Signed-off-by: CFC4N --- kern/go_argument.h | 4 ---- kern/gotls_kern.c | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/kern/go_argument.h b/kern/go_argument.h index 7779f9b6d..c590e77df 100644 --- a/kern/go_argument.h +++ b/kern/go_argument.h @@ -19,10 +19,6 @@ // false: newer //volatile const bool is_register_abi; -// // TLS record types in golang tls package -#define recordTypeApplicationData 23 - - // golang register-based ABI via https://tip.golang.org/src/cmd/compile/abi-internal #ifndef NOCORE diff --git a/kern/gotls_kern.c b/kern/gotls_kern.c index 82945799f..061d12ffb 100644 --- a/kern/gotls_kern.c +++ b/kern/gotls_kern.c @@ -23,6 +23,9 @@ #define MASTER_SECRET_KEY_LEN 32 #define EVP_MAX_MD_SIZE 64 +// // TLS record types in golang tls package +#define recordTypeApplicationData 23 + struct go_tls_event { u64 ts_ns; u32 pid;