Skip to content

Commit 071bfd8

Browse files
kmesh-botkmesh-bot
authored
Merge pull request #987 from lec-bit/kernel_native_restart
kernel-native mode support restart
2 parents e9455ba + 43f8129 commit 071bfd8

File tree

16 files changed

+257
-85
lines changed

16 files changed

+257
-85
lines changed

‎bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -889,6 +889,7 @@ static void *create_struct(struct op_context *ctx, int *err)
889889
}
890890

891891
ctx->value = value;
892+
((ProtobufCMessage *)value)->descriptor = desc;
892893
for (i = 0; i < desc->n_fields; i++) {
893894
const ProtobufCFieldDescriptor *field = desc->fields + i;
894895

‎pkg/bpf/ads/loader_enhanced.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,16 @@ type BpfAds struct {
4343

4444
func NewBpfAds(cfg *options.BpfConfig) (*BpfAds, error) {
4545
sc := &BpfAds{}
46-
sc.TracePoint.NewBpf(cfg)
46+
if err := sc.TracePoint.NewBpf(cfg); err != nil {
47+
return nil, err
48+
}
4749

4850
if err := sc.SockOps.NewBpf(cfg); err != nil {
49-
return sc, err
51+
return nil, err
5052
}
5153

5254
if err := sc.SockConn.NewBpf(cfg); err != nil {
53-
return sc, err
55+
return nil, err
5456
}
5557
return sc, nil
5658
}

‎pkg/bpf/ads/sock_connection.go‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,16 @@ package ads
2222
import "C"
2323
import (
2424
"os"
25+
"path/filepath"
2526
"reflect"
2627
"syscall"
2728

2829
"github.com/cilium/ebpf"
2930
"github.com/cilium/ebpf/link"
3031

32+
"kmesh.net/kmesh/pkg/bpf/restart"
33+
"kmesh.net/kmesh/pkg/constants"
34+
3135
bpf2go "kmesh.net/kmesh/bpf/kmesh/bpf2go/kernelnative/normal"
3236
"kmesh.net/kmesh/daemon/options"
3337
"kmesh.net/kmesh/pkg/bpf/utils"
@@ -98,11 +102,6 @@ func (sc *BpfSockConn) loadKmeshSockConnObjects() (*ebpf.CollectionSpec, error)
98102
return nil, err
99103
}
100104

101-
value := reflect.ValueOf(sc.KmeshCgroupSockObjects.KmeshCgroupSockPrograms)
102-
if err = utils.PinPrograms(&value, sc.Info.BpfFsPath); err != nil {
103-
return nil, err
104-
}
105-
106105
return spec, nil
107106
}
108107

@@ -154,18 +153,41 @@ func (sc *BpfSockConn) close() error {
154153
}
155154

156155
func (sc *BpfSockConn) Attach() error {
156+
var err error
157157
cgopt := link.CgroupOptions{
158158
Path: sc.Info.Cgroup2Path,
159159
Attach: sc.Info.AttachType,
160160
Program: sc.KmeshCgroupSockObjects.CgroupConnect4Prog,
161161
}
162162

163-
lk, err := link.AttachCgroup(cgopt)
164-
if err != nil {
163+
// pin bpf_tail_call map
164+
// tail_call map cannot pin in SetMapPinType->LoadAndAssign, we pin them independently
165+
// When we need to update tail_call map, delete the old map and then pin the new one.
166+
tailCallmapPinPath := filepath.Join(sc.Info.BpfFsPath, constants.TailCallMap)
167+
progPinPath := filepath.Join(sc.Info.BpfFsPath, constants.Prog_link)
168+
if restart.GetStartType() == restart.Restart {
169+
if sc.Link, err = utils.BpfProgUpdate(progPinPath, cgopt); err != nil {
170+
return err
171+
}
172+
173+
// Unpin tailcallmap. Considering that kmesh coredump may not have
174+
// this path after an unexpected restart, here we unpin the file by
175+
// directly removing it without doing error handling.
176+
os.Remove(tailCallmapPinPath)
177+
178+
} else {
179+
sc.Link, err = link.AttachCgroup(cgopt)
180+
if err != nil {
181+
return err
182+
}
183+
184+
if err := sc.Link.Pin(progPinPath); err != nil {
185+
return err
186+
}
187+
}
188+
if err = sc.KmeshCgroupSockMaps.KmeshTailCallProg.Pin(tailCallmapPinPath); err != nil {
165189
return err
166190
}
167-
sc.Link = lk
168-
169191
return nil
170192
}
171193

‎pkg/bpf/ads/sock_ops.go‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ package ads
2121

2222
import (
2323
"os"
24+
"path/filepath"
2425
"reflect"
2526
"syscall"
2627

@@ -29,7 +30,9 @@ import (
2930

3031
bpf2go "kmesh.net/kmesh/bpf/kmesh/bpf2go/kernelnative/enhanced"
3132
"kmesh.net/kmesh/daemon/options"
33+
"kmesh.net/kmesh/pkg/bpf/restart"
3234
"kmesh.net/kmesh/pkg/bpf/utils"
35+
"kmesh.net/kmesh/pkg/constants"
3336
helper "kmesh.net/kmesh/pkg/utils"
3437
)
3538

@@ -81,11 +84,6 @@ func (sc *BpfSockOps) loadKmeshSockopsObjects() (*ebpf.CollectionSpec, error) {
8184
return nil, err
8285
}
8386

84-
value := reflect.ValueOf(sc.KmeshSockopsObjects.KmeshSockopsPrograms)
85-
if err = utils.PinPrograms(&value, sc.Info.BpfFsPath); err != nil {
86-
return nil, err
87-
}
88-
8987
return spec, nil
9088
}
9189

@@ -180,18 +178,39 @@ func (sc *BpfSockOps) Load() error {
180178
}
181179

182180
func (sc *BpfSockOps) Attach() error {
181+
var err error
183182
cgopt := link.CgroupOptions{
184183
Path: sc.Info.Cgroup2Path,
185184
Attach: sc.Info.AttachType,
186185
Program: sc.KmeshSockopsObjects.SockopsProg,
187186
}
188187

189-
lk, err := link.AttachCgroup(cgopt)
190-
if err != nil {
188+
// pin bpf_link and bpf_tail_call map
189+
// pin bpf_link, after restart, update prog in bpf_link
190+
// tail_call map cannot pin in SetMapPinType->LoadAndAssign, we pin them independently
191+
// When we need to update tail_call map, delete the old map and then pin the new one.
192+
tailCallmapPinPath := filepath.Join(sc.Info.BpfFsPath, constants.TailCallMap)
193+
progPinPath := filepath.Join(sc.Info.BpfFsPath, constants.Prog_link)
194+
if restart.GetStartType() == restart.Restart {
195+
if sc.Link, err = utils.BpfProgUpdate(progPinPath, cgopt); err != nil {
196+
return err
197+
}
198+
// Unpin tailcallmap. Considering that kmesh coredump may not have
199+
// this path after an unexpected restart, here we unpin the file by
200+
// directly removing it without doing error handling.
201+
os.Remove(tailCallmapPinPath)
202+
} else {
203+
sc.Link, err = link.AttachCgroup(cgopt)
204+
if err != nil {
205+
return err
206+
}
207+
if err = sc.Link.Pin(progPinPath); err != nil {
208+
return err
209+
}
210+
}
211+
if err = sc.KmeshSockopsMaps.KmeshTailCallProg.Pin(tailCallmapPinPath); err != nil {
191212
return err
192213
}
193-
sc.Link = lk
194-
195214
return nil
196215
}
197216

‎pkg/bpf/ads/trace_point.go‎

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,17 @@
2020
package ads
2121

2222
import (
23+
"os"
24+
"path/filepath"
25+
"syscall"
26+
2327
"github.com/cilium/ebpf"
2428
"github.com/cilium/ebpf/link"
2529

2630
bpf2go "kmesh.net/kmesh/bpf/kmesh/bpf2go/kernelnative/enhanced"
2731
"kmesh.net/kmesh/daemon/options"
32+
"kmesh.net/kmesh/pkg/bpf/restart"
33+
"kmesh.net/kmesh/pkg/constants"
2834
helper "kmesh.net/kmesh/pkg/utils"
2935
)
3036

@@ -34,10 +40,24 @@ type BpfTracePoint struct {
3440
bpf2go.KmeshTracePointObjects
3541
}
3642

37-
func (sc *BpfTracePoint) NewBpf(cfg *options.BpfConfig) {
38-
sc.Info.MapPath = cfg.BpfFsPath
39-
sc.Info.BpfFsPath = cfg.BpfFsPath
43+
func (sc *BpfTracePoint) NewBpf(cfg *options.BpfConfig) error {
44+
sc.Info.MapPath = cfg.BpfFsPath + "/bpf_kmesh/map/"
45+
sc.Info.BpfFsPath = cfg.BpfFsPath + "/bpf_kmesh/tracepoint/"
4046
sc.Info.Cgroup2Path = cfg.Cgroup2Path
47+
48+
if err := os.MkdirAll(sc.Info.MapPath,
49+
syscall.S_IRUSR|syscall.S_IWUSR|syscall.S_IXUSR|
50+
syscall.S_IRGRP|syscall.S_IXGRP); err != nil && !os.IsExist(err) {
51+
return err
52+
}
53+
54+
if err := os.MkdirAll(sc.Info.BpfFsPath,
55+
syscall.S_IRUSR|syscall.S_IWUSR|syscall.S_IXUSR|
56+
syscall.S_IRGRP|syscall.S_IXGRP); err != nil && !os.IsExist(err) {
57+
return err
58+
}
59+
60+
return nil
4161
}
4262

4363
func (sc *BpfTracePoint) loadKmeshTracePointObjects() (*ebpf.CollectionSpec, error) {
@@ -76,17 +96,29 @@ func (sc *BpfTracePoint) Load() error {
7696
}
7797

7898
func (sc *BpfTracePoint) Attach() error {
99+
var err error
79100
tpopt := link.RawTracepointOptions{
80101
Name: "connect_ret",
81102
Program: sc.KmeshTracePointObjects.ConnectRet,
82103
}
83104

84-
lk, err := link.AttachRawTracepoint(tpopt)
85-
if err != nil {
86-
return err
87-
}
88-
sc.Link = lk
105+
pinPath := filepath.Join(sc.Info.BpfFsPath, constants.Prog_link)
106+
if restart.GetStartType() == restart.Restart {
107+
sc.Link, err = link.LoadPinnedLink(pinPath, &ebpf.LoadPinOptions{})
108+
if err != nil {
109+
return err
110+
}
111+
} else {
112+
sc.Link, err = link.AttachRawTracepoint(tpopt)
113+
if err != nil {
114+
return err
115+
}
116+
117+
if err := sc.Link.Pin(pinPath); err != nil {
118+
return err
119+
}
89120

121+
}
90122
return nil
91123
}
92124

‎pkg/bpf/bpf.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -157,14 +157,12 @@ func StopMda() error {
157157

158158
func (l *BpfLoader) Stop() {
159159
var err error
160-
if restart.GetExitType() == restart.Restart && l.config.DualEngineEnabled() {
161-
C.deserial_uninit()
162-
log.Infof("kmesh restart, not clean bpf map and prog")
160+
C.deserial_uninit()
161+
if restart.GetExitType() == restart.Restart {
163162
return
164163
}
165164

166165
closeMap(l.versionMap)
167-
168166
if l.config.KernelNativeEnabled() {
169167
if err = l.obj.Stop(); err != nil {
170168
CleanupBpfMap()

0 commit comments

Comments
 (0)