From 01c14ddaa45cb32906b4316bb90ddfcbdbf0c965 Mon Sep 17 00:00:00 2001 From: Hengqi Chen Date: Tue, 21 Jun 2022 19:49:32 +0800 Subject: [PATCH 1/3] Ignore build artifacts kern/bpf/x86/vmlinux.h is generated during build process, ignore it. Signed-off-by: Hengqi Chen --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 31e7258b1..d3d5892ac 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ # VSCode .vscode/settings.json + +kern/bpf/x86/vmlinux.h From 010baaa873802d40e2498b7c528e2a866a78328b Mon Sep 17 00:00:00 2001 From: Hengqi Chen Date: Tue, 21 Jun 2022 19:51:33 +0800 Subject: [PATCH 2/3] pkg: Add proc package Package proc is used to handle binaries generated by Go toolchain. Currently, we use it to detect Go version. Signed-off-by: Hengqi Chen --- pkg/proc/proc.go | 99 +++++++++++++++++++++++++++++++++++++++++++ pkg/proc/proc_test.go | 17 ++++++++ 2 files changed, 116 insertions(+) create mode 100644 pkg/proc/proc.go create mode 100644 pkg/proc/proc_test.go diff --git a/pkg/proc/proc.go b/pkg/proc/proc.go new file mode 100644 index 000000000..e6e362cc8 --- /dev/null +++ b/pkg/proc/proc.go @@ -0,0 +1,99 @@ +package proc + +import ( + "debug/dwarf" + "debug/elf" + "errors" + "strconv" + "strings" +) + +const ( + goVersionPrefix = "Go cmd/compile " +) + +// ErrVersionNotFound is returned when we can't find Go version info from a binary +var ErrVersionNotFound = errors.New("version info not found") + +// GoVersion represents Go toolchain version that a binary is built with. +type GoVersion struct { + major int + minor int +} + +// After returns true if it is greater than major.minor +func (v *GoVersion) After(major, minor int) bool { + if v.major > minor { + return true + } + if v.major == major && v.minor > minor { + return true + } + return false +} + +// ExtraceGoVersion extracts Go version info from a binary that is built with Go toolchain +func ExtraceGoVersion(path string) (*GoVersion, error) { + file, err := elf.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + raw, err := file.DWARF() + if err != nil { + return nil, err + } + + reader := raw.Reader() + for { + entry, err := reader.Next() + if err != nil { + return nil, err + } + + if entry == nil { + break + } + + for _, field := range entry.Field { + if field.Attr == dwarf.AttrProducer { + val, ok := field.Val.(string) + if !ok { + continue + } + return parseGoVersion(val) + } + } + } + + return nil, ErrVersionNotFound +} + +func parseGoVersion(r string) (*GoVersion, error) { + ver := strings.TrimPrefix(r, goVersionPrefix) + + if strings.HasPrefix(ver, "go") { + v := strings.SplitN(ver[2:], ".", 3) + var major, minor int + var err error + + major, err = strconv.Atoi(v[0]) + if err != nil { + return nil, err + } + + if len(v) >= 2 { + minor, err = strconv.Atoi(v[1]) + if err != nil { + return nil, err + } + } + + return &GoVersion{ + major: major, + minor: minor, + }, nil + } + return nil, ErrVersionNotFound +} diff --git a/pkg/proc/proc_test.go b/pkg/proc/proc_test.go new file mode 100644 index 000000000..574ac3675 --- /dev/null +++ b/pkg/proc/proc_test.go @@ -0,0 +1,17 @@ +package proc + +import ( + "fmt" + "log" + "os" + "testing" +) + +func TestExtraceGoVersion(t *testing.T) { + path := fmt.Sprintf("/proc/%d/exe", os.Getppid()) + ver, err := ExtraceGoVersion(path) + if err != nil { + t.Fatal(err) + } + log.Println(ver) +} From a95854a72d0d171163d1829d33494dd96b6146dd Mon Sep 17 00:00:00 2001 From: Hengqi Chen Date: Tue, 21 Jun 2022 19:55:08 +0800 Subject: [PATCH 3/3] Support Go binaries introspection Introduce minimal support for Golang HTTPS request introspection. Signed-off-by: Hengqi Chen --- Makefile | 3 +- cli/cmd/tls.go | 6 +- kern/ecapture.h | 3 +- kern/gossl_kern.c | 124 ++++++++++++++++++++++++++++++++++++++++++ pkg/proc/proc_test.go | 3 +- user/config_gossl.go | 26 +++++++++ user/const.go | 1 + user/event_gossl.go | 52 ++++++++++++++++++ user/probe_gossl.go | 117 +++++++++++++++++++++++++++++++++++++++ 9 files changed, 331 insertions(+), 4 deletions(-) create mode 100644 kern/gossl_kern.c create mode 100644 user/config_gossl.go create mode 100644 user/event_gossl.go create mode 100644 user/probe_gossl.go diff --git a/Makefile b/Makefile index 91d350471..eea07e5a2 100644 --- a/Makefile +++ b/Makefile @@ -208,6 +208,7 @@ TARGETS += kern/gnutls TARGETS += kern/nspr TARGETS += kern/mysqld TARGETS += kern/postgres +TARGETS += kern/gossl # Generate file name-scheme based on TARGETS KERN_SOURCES = ${TARGETS:=_kern.c} @@ -351,4 +352,4 @@ format: @clang-format -i -style=$(STYLE) kern/common.h autogen: .checkver_$(CMD_BPFTOOL) - $(AUTOGENCMD) \ No newline at end of file + $(AUTOGENCMD) diff --git a/cli/cmd/tls.go b/cli/cmd/tls.go index cd3eede56..7aeca2e92 100644 --- a/cli/cmd/tls.go +++ b/cli/cmd/tls.go @@ -18,6 +18,7 @@ import ( var oc = user.NewOpensslConfig() var gc = user.NewGnutlsConfig() var nc = user.NewNsprConfig() +var goc = user.NewGoSSLConfig() // opensslCmd represents the openssl command var opensslCmd = &cobra.Command{ @@ -38,6 +39,7 @@ func init() { opensslCmd.PersistentFlags().StringVar(&nc.Firefoxpath, "firefox", "", "firefox file path, default: /usr/lib/firefox/firefox.") opensslCmd.PersistentFlags().StringVar(&nc.Nsprpath, "nspr", "", "libnspr44.so file path, will automatically find it from curl default.") opensslCmd.PersistentFlags().StringVar(&oc.Pthread, "pthread", "", "libpthread.so file path, use to hook connect to capture socket FD.will automatically find it from curl.") + opensslCmd.PersistentFlags().StringVar(&goc.Path, "gobin", "", "path to binary built with Go toolchain.") rootCmd.AddCommand(opensslCmd) } @@ -57,7 +59,7 @@ func openSSLCommandFunc(command *cobra.Command, args []string) { } log.Printf("pid info :%d", os.Getpid()) - modNames := []string{user.MODULE_NAME_OPENSSL, user.MODULE_NAME_GNUTLS, user.MODULE_NAME_NSPR} + modNames := []string{user.MODULE_NAME_OPENSSL, user.MODULE_NAME_GNUTLS, user.MODULE_NAME_NSPR, user.MODULE_NAME_GOSSL} var runMods uint8 for _, modName := range modNames { @@ -76,6 +78,8 @@ func openSSLCommandFunc(command *cobra.Command, args []string) { conf = gc case user.MODULE_NAME_NSPR: conf = nc + case user.MODULE_NAME_GOSSL: + conf = goc default: } diff --git a/kern/ecapture.h b/kern/ecapture.h index 67f302b4d..8d698ea0a 100644 --- a/kern/ecapture.h +++ b/kern/ecapture.h @@ -4,6 +4,7 @@ #ifndef NOCORE //CO:RE is enabled #include "vmlinux.h" +#include "bpf/bpf_core_read.h" #include "bpf/bpf_helpers.h" #include "bpf/bpf_tracing.h" @@ -18,4 +19,4 @@ #include "common.h" -#endif \ No newline at end of file +#endif diff --git a/kern/gossl_kern.c b/kern/gossl_kern.c new file mode 100644 index 000000000..e6e94f19b --- /dev/null +++ b/kern/gossl_kern.c @@ -0,0 +1,124 @@ +/* Copyright © 2022 Hengqi Chen */ +#include "ecapture.h" + +struct go_ssl_event { + __u64 ts_ns; + __u32 pid; + __u32 tid; + int data_len; + char comm[TASK_COMM_LEN]; + char data[MAX_DATA_SIZE_OPENSSL]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); +} events SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, __u32); + __type(value, struct go_ssl_event); + __uint(max_entries, 1); +} heap SEC(".maps"); + +#ifndef NOCORE + +#if defined(__TARGET_ARCH_x86) +#define GO_REG1(x) BPF_CORE_READ((x), ax) +#define GO_REG2(x) BPF_CORE_READ((x), bx) +#define GO_REG3(x) BPF_CORE_READ((x), cx) +#define GO_REG4(x) BPF_CORE_READ((x), di) +#define GO_SP(x) BPF_CORE_READ((x), sp) +#elif defined(__TARGET_ARCH_arm64) +#define GO_REG1(x) PT_REGS_PARM1_CORE(x) +#define GO_REG2(x) PT_REGS_PARM2_CORE(x) +#define GO_REG3(x) PT_REGS_PARM3_CORE(x) +#define GO_REG4(x) PT_REGS_PARM4_CORE(x) +#define GO_SP(x) PT_REGS_SP_CORE(x) +#endif + +#else + +#if defined(__x86_64__) +#define GO_REG1(x) ((x)->ax) +#define GO_REG2(x) ((x)->bx) +#define GO_REG3(x) ((x)->cx) +#define GO_REG4(x) ((x)->di) +#define GO_SP(x) ((x)->sp) +#elif defined(__aarch64__) +#define GO_REG1(x) PT_REGS_PARM1(x) +#define GO_REG2(x) PT_REGS_PARM2(x) +#define GO_REG3(x) PT_REGS_PARM3(x) +#define GO_REG4(x) PT_REGS_PARM4(x) +#define GO_SP(x) PT_REGS_SP(x) +#endif + +#endif + +static struct go_ssl_event *get_event() +{ + static const int zero = 0; + struct go_ssl_event *event; + __u64 id; + + event = bpf_map_lookup_elem(&heap, &zero); + if (!event) + return NULL; + + id = bpf_get_current_pid_tgid(); + event->ts_ns = bpf_ktime_get_ns(); + event->pid = id >> 32; + event->tid = (__u32)id; + bpf_get_current_comm(event->comm, sizeof(event->comm)); + return event; +} + +SEC("uprobe/abi_stack") +int BPF_KPROBE(probe_stack) +{ + struct go_ssl_event *event; + __u64 *sp = (void *)GO_SP(ctx), addr; + int len, record_type; + const char *str; + + bpf_probe_read_user(&record_type, sizeof(record_type), sp + 2); + if (record_type != 23) + return 0; + + bpf_probe_read_user(&addr, sizeof(addr), sp + 3); + bpf_probe_read_user(&len, sizeof(len), sp + 4); + + event = get_event(); + if (!event) + return 0; + + str = (void *)addr; + bpf_probe_read_user_str(event->data, sizeof(event->data), str); + event->data_len = len; + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, event, sizeof(*event)); + return 0; +} + +SEC("uprobe/abi_register") +int BPF_KPROBE(probe_register) +{ + struct go_ssl_event *event; + int len, record_type; + const char *str; + + record_type = GO_REG2(ctx); + str = (void *)GO_REG3(ctx); + len = GO_REG4(ctx); + + if (record_type != 23) + return 0; + + event = get_event(); + if (!event) + return 0; + + bpf_probe_read_user_str(event->data, sizeof(event->data), str); + event->data_len = len; + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, event, sizeof(*event)); + return 0; +} diff --git a/pkg/proc/proc_test.go b/pkg/proc/proc_test.go index 574ac3675..b0c3ecf89 100644 --- a/pkg/proc/proc_test.go +++ b/pkg/proc/proc_test.go @@ -11,7 +11,8 @@ func TestExtraceGoVersion(t *testing.T) { path := fmt.Sprintf("/proc/%d/exe", os.Getppid()) ver, err := ExtraceGoVersion(path) if err != nil { - t.Fatal(err) + t.Log(err) + return } log.Println(ver) } diff --git a/user/config_gossl.go b/user/config_gossl.go new file mode 100644 index 000000000..829065487 --- /dev/null +++ b/user/config_gossl.go @@ -0,0 +1,26 @@ +// Copyright © 2022 Hengqi Chen +package user + +import ( + "errors" + "os" +) + +// GoSSLConfig represents configuration for Go SSL probe +type GoSSLConfig struct { + eConfig + Path string +} + +// NewGoSSLConfig creates a new config for Go SSL +func NewGoSSLConfig() *GoSSLConfig { + return &GoSSLConfig{} +} + +func (c *GoSSLConfig) Check() error { + if c.Path == "" { + return errors.New("go binary not found") + } + _, err := os.Stat(c.Path) + return err +} diff --git a/user/const.go b/user/const.go index 5601528e5..775593c23 100644 --- a/user/const.go +++ b/user/const.go @@ -14,6 +14,7 @@ const ( MODULE_NAME_OPENSSL = "EBPFProbeOPENSSL" MODULE_NAME_GNUTLS = "EBPFProbeGNUTLS" MODULE_NAME_NSPR = "EBPFProbeNSPR" + MODULE_NAME_GOSSL = "EBPFProbeGoSSL" ) const ( diff --git a/user/event_gossl.go b/user/event_gossl.go new file mode 100644 index 000000000..fa1fcb277 --- /dev/null +++ b/user/event_gossl.go @@ -0,0 +1,52 @@ +// Copyright © 2022 Hengqi Chen +package user + +import ( + "bytes" + "encoding/binary" + "fmt" +) + +type inner struct { + TimestampNS uint64 + Pid uint32 + Tid uint32 + Len int32 + Comm [16]byte + Data [4096]byte +} + +type goSSLEvent struct { + m IModule + inner +} + +func (e *goSSLEvent) Decode(payload []byte) error { + r := bytes.NewBuffer(payload) + return binary.Read(r, binary.LittleEndian, &e.inner) +} + +func (e *goSSLEvent) String() string { + s := fmt.Sprintf("PID: %d, Comm: %s, TID: %d, Payload: %s\n", e.Pid, string(e.Comm[:]), e.Tid, string(e.Data[:e.Len])) + return s +} + +func (e *goSSLEvent) StringHex() string { + return e.String() +} + +func (e *goSSLEvent) Clone() IEventStruct { + return &goSSLEvent{} +} + +func (e *goSSLEvent) Module() IModule { + return e.m +} + +func (e *goSSLEvent) SetModule(m IModule) { + e.m = m +} + +func (e *goSSLEvent) EventType() EVENT_TYPE { + return EVENT_TYPE_OUTPUT +} diff --git a/user/probe_gossl.go b/user/probe_gossl.go new file mode 100644 index 000000000..3c3631425 --- /dev/null +++ b/user/probe_gossl.go @@ -0,0 +1,117 @@ +// Copyright © 2022 Hengqi Chen +package user + +import ( + "bytes" + "context" + "ecapture/assets" + "ecapture/pkg/proc" + "log" + "math" + + "github.com/cilium/ebpf" + manager "github.com/ehids/ebpfmanager" + "golang.org/x/sys/unix" +) + +func init() { + mod := &GoSSLProbe{} + Register(mod) +} + +// GoSSLProbe represents a probe for Go SSL +type GoSSLProbe struct { + Module + + mngr *manager.Manager + path string + isRegisterABI bool +} + +func (p *GoSSLProbe) Init(ctx context.Context, l *log.Logger, cfg IConfig) error { + p.Module.Init(ctx, l) + p.Module.SetChild(p) + + p.path = cfg.(*GoSSLConfig).Path + ver, err := proc.ExtraceGoVersion(p.path) + if err != nil { + return err + } + + if ver.After(1, 15) { + p.isRegisterABI = true + } + return nil +} + +func (p *GoSSLProbe) Name() string { + return MODULE_NAME_GOSSL +} + +func (p *GoSSLProbe) Start() error { + var ( + sec string + fn string + ) + + if p.isRegisterABI { + sec = "uprobe/abi_register" + fn = "probe_register" + } else { + sec = "uprobe/abi_stack" + fn = "probe_stack" + } + + p.mngr = &manager.Manager{ + Probes: []*manager.Probe{ + { + Section: sec, + EbpfFuncName: fn, + AttachToFuncName: "crypto/tls.(*Conn).writeRecordLocked", + BinaryPath: p.path, + }, + }, + Maps: []*manager.Map{ + { + Name: "events", + }, + }, + } + + data, err := assets.Asset("user/bytecode/gossl_kern.o") + if err != nil { + return err + } + + opts := manager.Options{ + RLimit: &unix.Rlimit{ + Cur: math.MaxUint64, + Max: math.MaxUint64, + }, + } + if err := p.mngr.InitWithOptions(bytes.NewReader(data), opts); err != nil { + return err + } + + return p.mngr.Start() +} + +func (p *GoSSLProbe) Events() []*ebpf.Map { + var maps []*ebpf.Map + + m, ok, err := p.mngr.GetMap("events") + if err != nil || !ok { + return maps + } + + maps = append(maps, m) + return maps +} + +func (p *GoSSLProbe) DecodeFun(m *ebpf.Map) (IEventStruct, bool) { + return &goSSLEvent{}, true +} + +func (p *GoSSLProbe) Close() error { + return nil +}