diff --git a/btf/format.go b/btf/format.go index 5e581b4a8..6b07c3444 100644 --- a/btf/format.go +++ b/btf/format.go @@ -4,10 +4,45 @@ import ( "errors" "fmt" "strings" + "text/template" + + "golang.org/x/text/cases" + "golang.org/x/text/language" ) var errNestedTooDeep = errors.New("nested too deep") +var tplVars = ` + +type {{ .Name }} ebpf.Variable + +func(v *{{ .Name }}) Get() ({{ .Type }}, error) { + var ret {{ .Type }} + return ret, (*ebpf.Variable)(v).Get(&ret) +} + +{{- if not .ReadOnly }} +func(v *{{ .Name }}) Set(val {{ .Type }}) error { + return (*ebpf.Variable)(v).Set(val) +} + +{{ if .CanAtomic }} +func(v *{{ .Name }}) AtomicRef() *ebpf.{{ .CapitalizedType }} { + ret, _ := (*ebpf.Variable)(v).Atomic{{ .CapitalizedType }}() + return ret +} +{{ end }} +{{- end }} +` + +type TplVarsData struct { + Name string + Type string + CapitalizedType string + ReadOnly bool + CanAtomic bool +} + // GoFormatter converts a Type to Go syntax. // // A zero GoFormatter is valid to use. @@ -64,7 +99,12 @@ func (gf *GoFormatter) writeTypeDecl(name string, typ Type) error { } typ = skipQualifiers(typ) - fmt.Fprintf(&gf.w, "type %s ", name) + // custom handling Datasec types in writeDatasecLit + _, ok := typ.(*Datasec) + if !ok { + fmt.Fprintf(&gf.w, "type %s ", name) + } + if err := gf.writeTypeLit(typ, 0); err != nil { return err } @@ -295,41 +335,82 @@ func (gf *GoFormatter) writeStructField(m Member, depth int) error { } func (gf *GoFormatter) writeDatasecLit(ds *Datasec, depth int) error { - gf.w.WriteString("struct { ") + tmpl, err := template.New("varsHelpers").Parse(tplVars) + if err != nil { + return fmt.Errorf("failed to parse template: %w", err) + } - prevOffset := uint32(0) for i, vsi := range ds.Vars { v, ok := vsi.Type.(*Var) if !ok { return fmt.Errorf("can't format %s as part of data section", vsi.Type) } - if v.Linkage != GlobalVar { - // Ignore static, extern, etc. for now. - continue - } - - if v.Name == "" { - return fmt.Errorf("variable %d: empty name", i) + id := gf.identifier(v.Name) + va := getSuppAtomicType(v.Type) + tplArgs := TplVarsData{ + Name: id, + Type: id + "Type", + CapitalizedType: cases.Title(language.Und, cases.NoLower).String(va), + ReadOnly: strings.HasPrefix(ds.Name, ".ro"), + CanAtomic: va != "", } - gf.writePadding(vsi.Offset - prevOffset) - prevOffset = vsi.Offset + vsi.Size - - fmt.Fprintf(&gf.w, "%s ", gf.identifier(v.Name)) + fmt.Fprintf(&gf.w, "type %s =", tplArgs.Type) if err := gf.writeType(v.Type, depth); err != nil { return fmt.Errorf("variable %d: %w", i, err) } - gf.w.WriteString("; ") + if err := tmpl.Execute(&gf.w, tplArgs); err != nil { + return fmt.Errorf("failed to execute template for variable %s: %w", tplArgs.Name, err) + } } - gf.writePadding(ds.Size - prevOffset) - gf.w.WriteString("}") return nil } +// getSuppAtomicType returns the corresponding Go type for the +// provided argument if it supports package atomic primitives. +// Current support for int32, uint32, int64, uint64. +func getSuppAtomicType(t Type) string { + checkInt := func(t *Int) string { + ret := "" + switch t.Size { + // uint32/int32 and uint64/int64 + case 4: + ret = "int32" + case 8: + ret = "int64" + default: + return "" + } + if t.Encoding == Unsigned { + ret = "u" + ret + } + return ret + } + + switch v := skipQualifiers(t).(type) { + case *Int: + return checkInt(v) + case *Typedef: + if vv, ok := v.Type.(*Int); ok { + return checkInt(vv) + } + case *Enum: + i := &Int{ + Name: v.Name, + Size: v.Size, + Encoding: Unsigned, + } + if v.Signed { + i.Encoding = Signed + } + return checkInt(i) + } + return "" +} func (gf *GoFormatter) writePadding(bytes uint32) { if bytes > 0 { fmt.Fprintf(&gf.w, "_ [%d]byte; ", bytes) diff --git a/cmd/bpf2go/gen/output.go b/cmd/bpf2go/gen/output.go index a054fd2f1..2ade96375 100644 --- a/cmd/bpf2go/gen/output.go +++ b/cmd/bpf2go/gen/output.go @@ -49,6 +49,10 @@ func (n templateName) MapSpecs() string { return string(n) + "MapSpecs" } +func (n templateName) VariableSpecs() string { + return string(n) + "VariableSpecs" +} + func (n templateName) Load() string { return n.maybeExport("load" + toUpperFirst(string(n))) } @@ -65,6 +69,10 @@ func (n templateName) Maps() string { return string(n) + "Maps" } +func (n templateName) Variables() string { + return string(n) + "Variables" +} + func (n templateName) Programs() string { return string(n) + "Programs" } @@ -82,6 +90,8 @@ type GenerateArgs struct { Constraints constraint.Expr // Maps to be emitted. Maps []string + // Variables to be emitted. + Variables []string // Programs to be emitted. Programs []string // Types to be emitted. @@ -103,19 +113,16 @@ func Generate(args GenerateArgs) error { return fmt.Errorf("file %q contains an invalid character", args.ObjectFile) } - for _, typ := range args.Types { - if _, ok := btf.As[*btf.Datasec](typ); ok { - // Avoid emitting .rodata, .bss, etc. for now. We might want to - // name these types differently, etc. - return fmt.Errorf("can't output btf.Datasec: %s", typ) - } - } - maps := make(map[string]string) for _, name := range args.Maps { maps[name] = internal.Identifier(name) } + vars := make(map[string]string) + for _, name := range args.Variables { + vars[name] = internal.Identifier(name) + } + programs := make(map[string]string) for _, name := range args.Programs { programs[name] = internal.Identifier(name) @@ -146,6 +153,7 @@ func Generate(args GenerateArgs) error { Constraints constraint.Expr Name templateName Maps map[string]string + Variables map[string]string Programs map[string]string Types []btf.Type TypeNames map[btf.Type]string @@ -157,6 +165,7 @@ func Generate(args GenerateArgs) error { args.Constraints, templateName(args.Stem), maps, + vars, programs, types, typeNames, diff --git a/cmd/bpf2go/gen/output.tpl b/cmd/bpf2go/gen/output.tpl index 8d8047066..d5e2c2e8a 100644 --- a/cmd/bpf2go/gen/output.tpl +++ b/cmd/bpf2go/gen/output.tpl @@ -54,6 +54,7 @@ func {{ .Name.LoadObjects }}(obj interface{}, opts *ebpf.CollectionOptions) (err type {{ .Name.Specs }} struct { {{ .Name.ProgramSpecs }} {{ .Name.MapSpecs }} + {{ .Name.VariableSpecs }} } // {{ .Name.Specs }} contains programs before they are loaded into the kernel. @@ -80,6 +81,7 @@ type {{ .Name.MapSpecs }} struct { type {{ .Name.Objects }} struct { {{ .Name.Programs }} {{ .Name.Maps }} + {{ .Name.Variables }} } func (o *{{ .Name.Objects }}) Close() error { @@ -98,6 +100,24 @@ type {{ .Name.Maps }} struct { {{- end }} } +// {{ .Name.VariableSpecs }} contains variables before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type {{ .Name.VariableSpecs }} struct { +{{- range $name, $id := .Variables }} + {{ $id }} *ebpf.VariableSpec `ebpf:"{{ $name }}"` +{{- end }} +} + +// {{ .Name.Variables }} contains all variables after they have been loaded into the kernel. +// +// It can be passed to {{ .Name.LoadObjects }} or ebpf.CollectionSpec.LoadAndAssign. +type {{ .Name.Variables }} struct { +{{- range $name, $id := .Variables }} + {{ $id }} *{{ $id }} `ebpf:"{{ $name }}"` +{{- end }} +} + func (m *{{ .Name.Maps }}) Close() error { return {{ .Name.CloseHelper }}( {{- range $id := .Maps }} diff --git a/cmd/bpf2go/gen/types.go b/cmd/bpf2go/gen/types.go index 37dad0c76..2973c17a7 100644 --- a/cmd/bpf2go/gen/types.go +++ b/cmd/bpf2go/gen/types.go @@ -12,10 +12,6 @@ func CollectGlobalTypes(spec *ebpf.CollectionSpec) []btf.Type { var types []btf.Type for _, typ := range collectMapTypes(spec.Maps) { switch btf.UnderlyingType(typ).(type) { - case *btf.Datasec: - // Avoid emitting .rodata, .bss, etc. for now. We might want to - // name these types differently, etc. - continue case *btf.Int: // Don't emit primitive types by default. diff --git a/cmd/bpf2go/main.go b/cmd/bpf2go/main.go index fb077e139..a5164ff5e 100644 --- a/cmd/bpf2go/main.go +++ b/cmd/bpf2go/main.go @@ -370,6 +370,11 @@ func (b2g *bpf2go) convert(tgt gen.Target, goarches gen.GoArches) (err error) { } } + var vars []string + for name := range spec.Variables { + vars = append(vars, name) + } + var programs []string for name := range spec.Programs { programs = append(programs, name) @@ -397,6 +402,7 @@ func (b2g *bpf2go) convert(tgt gen.Target, goarches gen.GoArches) (err error) { Stem: b2g.identStem, Constraints: constraints, Maps: maps, + Variables: vars, Programs: programs, Types: types, ObjectFile: filepath.Base(objFileName), diff --git a/collection.go b/collection.go index 36231056f..79ba3fd76 100644 --- a/collection.go +++ b/collection.go @@ -12,7 +12,6 @@ import ( "github.com/cilium/ebpf/internal" "github.com/cilium/ebpf/internal/kconfig" "github.com/cilium/ebpf/internal/linux" - "github.com/cilium/ebpf/internal/sysenc" ) // CollectionOptions control loading a collection into the kernel. @@ -36,8 +35,9 @@ type CollectionOptions struct { // CollectionSpec describes a collection. type CollectionSpec struct { - Maps map[string]*MapSpec - Programs map[string]*ProgramSpec + Maps map[string]*MapSpec + Programs map[string]*ProgramSpec + Variables map[string]*VariableSpec // Types holds type information about Maps and Programs. // Modifications to Types are currently undefined behaviour. @@ -57,6 +57,7 @@ func (cs *CollectionSpec) Copy() *CollectionSpec { cpy := CollectionSpec{ Maps: make(map[string]*MapSpec, len(cs.Maps)), Programs: make(map[string]*ProgramSpec, len(cs.Programs)), + Variables: make(map[string]*VariableSpec, len(cs.Variables)), ByteOrder: cs.ByteOrder, Types: cs.Types.Copy(), } @@ -69,6 +70,10 @@ func (cs *CollectionSpec) Copy() *CollectionSpec { cpy.Programs[name] = spec.Copy() } + for name, spec := range cs.Variables { + cpy.Variables[name] = spec.copy(&cpy) + } + return &cpy } @@ -135,65 +140,24 @@ func (m *MissingConstantsError) Error() string { // From Linux 5.5 the verifier will use constants to eliminate dead code. // // Returns an error wrapping [MissingConstantsError] if a constant doesn't exist. +// +// Deprecated: Use [CollectionSpec.Variables] to interact with constants instead. +// RewriteConstants is now a wrapper around the VariableSpec API. func (cs *CollectionSpec) RewriteConstants(consts map[string]interface{}) error { - replaced := make(map[string]bool) - - for name, spec := range cs.Maps { - if !strings.HasPrefix(name, ".rodata") { + var missing []string + for n, c := range consts { + v, ok := cs.Variables[n] + if !ok { + missing = append(missing, n) continue } - b, ds, err := spec.dataSection() - if errors.Is(err, errMapNoBTFValue) { - // Data sections without a BTF Datasec are valid, but don't support - // constant replacements. - continue - } - if err != nil { - return fmt.Errorf("map %s: %w", name, err) + if !v.Constant() { + return fmt.Errorf("variable %s is not a constant", n) } - // MapSpec.Copy() performs a shallow copy. Fully copy the byte slice - // to avoid any changes affecting other copies of the MapSpec. - cpy := make([]byte, len(b)) - copy(cpy, b) - - for _, v := range ds.Vars { - vname := v.Type.TypeName() - replacement, ok := consts[vname] - if !ok { - continue - } - - if _, ok := v.Type.(*btf.Var); !ok { - return fmt.Errorf("section %s: unexpected type %T for variable %s", name, v.Type, vname) - } - - if replaced[vname] { - return fmt.Errorf("section %s: duplicate variable %s", name, vname) - } - - if int(v.Offset+v.Size) > len(cpy) { - return fmt.Errorf("section %s: offset %d(+%d) for variable %s is out of bounds", name, v.Offset, v.Size, vname) - } - - b, err := sysenc.Marshal(replacement, int(v.Size)) - if err != nil { - return fmt.Errorf("marshaling constant replacement %s: %w", vname, err) - } - - b.CopyTo(cpy[v.Offset : v.Offset+v.Size]) - - replaced[vname] = true - } - - spec.Contents[0] = MapKV{Key: uint32(0), Value: cpy} - } - - var missing []string - for c := range consts { - if !replaced[c] { - missing = append(missing, c) + if err := v.Set(c); err != nil { + return fmt.Errorf("rewriting constant %s: %w", n, err) } } @@ -225,8 +189,8 @@ func (cs *CollectionSpec) RewriteConstants(consts map[string]interface{}) error // Returns an error if any of the eBPF objects can't be found, or // if the same MapSpec or ProgramSpec is assigned multiple times. func (cs *CollectionSpec) Assign(to interface{}) error { - // Assign() only supports assigning ProgramSpecs and MapSpecs, - // so doesn't load any resources into the kernel. + // Assign() only supports assigning ProgramSpecs, MapSpecs + // and VariableSpecs, so doesn't load any resources into the kernel. getValue := func(typ reflect.Type, name string) (interface{}, error) { switch typ { @@ -242,6 +206,12 @@ func (cs *CollectionSpec) Assign(to interface{}) error { } return nil, fmt.Errorf("missing map %q", name) + case reflect.TypeOf((*VariableSpec)(nil)): + if m := cs.Variables[name]; m != nil { + return m, nil + } + return nil, fmt.Errorf("missing variable %q", name) + default: return nil, fmt.Errorf("unsupported type %s", typ) } @@ -287,8 +257,14 @@ func (cs *CollectionSpec) LoadAndAssign(to interface{}, opts *CollectionOptions) // Support assigning Programs and Maps, lazy-loading the required objects. assignedMaps := make(map[string]bool) assignedProgs := make(map[string]bool) + assignedVars := make(map[string]bool) getValue := func(typ reflect.Type, name string) (interface{}, error) { + handleVar := func() (*Variable, error) { + assignedVars[name] = true + return loader.loadVariable(name) + } + switch typ { case reflect.TypeOf((*Program)(nil)): @@ -299,7 +275,13 @@ func (cs *CollectionSpec) LoadAndAssign(to interface{}, opts *CollectionOptions) assignedMaps[name] = true return loader.loadMap(name) + case reflect.TypeOf((*Variable)(nil)): + return handleVar() + default: + if reflect.TypeOf((*Variable)(nil)).ConvertibleTo(typ) { + return handleVar() + } return nil, fmt.Errorf("unsupported type %s", typ) } } @@ -339,15 +321,22 @@ func (cs *CollectionSpec) LoadAndAssign(to interface{}, opts *CollectionOptions) for p := range assignedProgs { delete(loader.programs, p) } + for p := range assignedVars { + delete(loader.vars, p) + } return nil } -// Collection is a collection of Programs and Maps associated -// with their symbols +// Collection is a collection of live BPF resources present in the kernel. type Collection struct { Programs map[string]*Program Maps map[string]*Map + + // Variables contains global variables used by the Collection's program(s). + // Only populated on Linux 5.5 and later or on kernels supporting + // BPF_F_MMAPABLE. + Variables map[string]*Variable } // NewCollection creates a Collection from the given spec, creating and @@ -388,19 +377,26 @@ func NewCollectionWithOptions(spec *CollectionSpec, opts CollectionOptions) (*Co } } + for varName := range spec.Variables { + if _, err := loader.loadVariable(varName); err != nil { + return nil, err + } + } + // Maps can contain Program and Map stubs, so populate them after // all Maps and Programs have been successfully loaded. if err := loader.populateDeferredMaps(); err != nil { return nil, err } - // Prevent loader.cleanup from closing maps and programs. - maps, progs := loader.maps, loader.programs - loader.maps, loader.programs = nil, nil + // Prevent loader.cleanup from closing maps, vars and programs. + maps, progs, vars := loader.maps, loader.programs, loader.vars + loader.maps, loader.programs, loader.vars = nil, nil, nil return &Collection{ progs, maps, + vars, }, nil } @@ -409,6 +405,7 @@ type collectionLoader struct { opts *CollectionOptions maps map[string]*Map programs map[string]*Program + vars map[string]*Variable } func newCollectionLoader(coll *CollectionSpec, opts *CollectionOptions) (*collectionLoader, error) { @@ -433,6 +430,7 @@ func newCollectionLoader(coll *CollectionSpec, opts *CollectionOptions) (*collec opts, make(map[string]*Map), make(map[string]*Program), + make(map[string]*Variable), }, nil } @@ -538,6 +536,51 @@ func (cl *collectionLoader) loadProgram(progName string) (*Program, error) { return prog, nil } +func (cl *collectionLoader) loadVariable(varName string) (*Variable, error) { + if v := cl.vars[varName]; v != nil { + return v, nil + } + + varSpec := cl.coll.Variables[varName] + if varSpec == nil { + return nil, fmt.Errorf("unknown variable %s", varName) + } + + // Get the key of the VariableSpec's MapSpec in the CollectionSpec. + var mapName string + for n, ms := range cl.coll.Maps { + if ms == varSpec.m { + mapName = n + break + } + } + if mapName == "" { + return nil, fmt.Errorf("variable %s: underlying MapSpec %s was removed from CollectionSpec", varName, varSpec.m.Name) + } + + m, err := cl.loadMap(mapName) + if err != nil { + return nil, fmt.Errorf("variable %s: %w", varName, err) + } + + mm, err := m.Memory() + if err != nil { + return nil, fmt.Errorf("variable %s: getting memory of map %s: %w", varName, mapName, err) + } + + v := &Variable{ + varSpec.name, + varSpec.offset, + varSpec.size, + varSpec.Constant(), + mm, + varSpec.t, + } + + cl.vars[varName] = v + return v, nil +} + // populateDeferredMaps iterates maps holding programs or other maps and loads // any dependencies. Populates all maps in cl and freezes them if specified. func (cl *collectionLoader) populateDeferredMaps() error { @@ -724,10 +767,19 @@ func LoadCollection(file string) (*Collection, error) { func (coll *Collection) Assign(to interface{}) error { assignedMaps := make(map[string]bool) assignedProgs := make(map[string]bool) + assignedVars := make(map[string]bool) - // Assign() only transfers already-loaded Maps and Programs. No extra - // loading is done. + // Assign() only transfers already-loaded Maps, Programs and Variablees. + // No extra loading is done. getValue := func(typ reflect.Type, name string) (interface{}, error) { + handleVar := func() (*Variable, error) { + if v := coll.Variables[name]; v != nil { + assignedVars[name] = true + return v, nil + } + return nil, fmt.Errorf("missing variable %q", name) + } + switch typ { case reflect.TypeOf((*Program)(nil)): @@ -744,7 +796,14 @@ func (coll *Collection) Assign(to interface{}) error { } return nil, fmt.Errorf("missing map %q", name) + case reflect.TypeOf((*Variable)(nil)): + return handleVar() + default: + if reflect.TypeOf((*Variable)(nil)).ConvertibleTo(typ) { + handleVar() + } + return nil, fmt.Errorf("unsupported type %s", typ) } } @@ -760,6 +819,9 @@ func (coll *Collection) Assign(to interface{}) error { for m := range assignedMaps { delete(coll.Maps, m) } + for s := range assignedVars { + delete(coll.Variables, s) + } return nil } @@ -917,9 +979,23 @@ func assignValues(to interface{}, if !field.value.CanSet() { return fmt.Errorf("field %s: can't set value", field.Name) } - field.value.Set(reflect.ValueOf(value)) - assigned[e] = field.Name + // Match equal variables assignments. + if field.Type == reflect.TypeOf(value) { + field.value.Set(reflect.ValueOf(value)) + assigned[e] = field.Name + continue + } + + // Match all wrappers around `value` that still can convert to `field.Type`. + // Example: having an `int` variable assigned to `type MyType int`. + if reflect.TypeOf(value).ConvertibleTo(field.Type) { + field.value.Set(reflect.ValueOf(value).Convert(field.Type)) + assigned[e] = field.Name + continue + } + + panic(fmt.Sprintf("unable to assign type %v to type %v", reflect.TypeOf(value), field.Type)) } return nil diff --git a/collection_test.go b/collection_test.go index c82c3acc9..41bffd1b4 100644 --- a/collection_test.go +++ b/collection_test.go @@ -57,15 +57,15 @@ func TestCollectionSpecNotModified(t *testing.T) { } func TestCollectionSpecCopy(t *testing.T) { + ms := &MapSpec{ + Type: Array, + KeySize: 4, + ValueSize: 4, + MaxEntries: 1, + } + cs := &CollectionSpec{ - map[string]*MapSpec{ - "my-map": { - Type: Array, - KeySize: 4, - ValueSize: 4, - MaxEntries: 1, - }, - }, + map[string]*MapSpec{"my-map": ms}, map[string]*ProgramSpec{ "test": { Type: SocketFilter, @@ -77,6 +77,14 @@ func TestCollectionSpecCopy(t *testing.T) { License: "MIT", }, }, + map[string]*VariableSpec{ + "my-var": { + name: "my-var", + offset: 0, + size: 4, + m: ms, + }, + }, &btf.Spec{}, binary.LittleEndian, } @@ -325,52 +333,6 @@ func TestCollectionSpecMapReplacements_SpecMismatch(t *testing.T) { } } -func TestCollectionRewriteConstants(t *testing.T) { - cs := &CollectionSpec{ - Maps: map[string]*MapSpec{ - ".rodata": { - Type: Array, - KeySize: 4, - ValueSize: 4, - MaxEntries: 1, - Value: &btf.Datasec{ - Vars: []btf.VarSecinfo{ - { - Type: &btf.Var{ - Name: "the_constant", - Type: &btf.Int{Size: 4}, - }, - Offset: 0, - Size: 4, - }, - }, - }, - Contents: []MapKV{ - {Key: uint32(0), Value: []byte{1, 1, 1, 1}}, - }, - }, - }, - } - - err := cs.RewriteConstants(map[string]interface{}{ - "fake_constant_one": uint32(1), - "fake_constant_two": uint32(2), - }) - qt.Assert(t, qt.IsNotNil(err), qt.Commentf("RewriteConstants did not fail")) - - var mErr *MissingConstantsError - if !errors.As(err, &mErr) { - t.Fatal("Error doesn't wrap MissingConstantsError:", err) - } - qt.Assert(t, qt.ContentEquals(mErr.Constants, []string{"fake_constant_one", "fake_constant_two"})) - - err = cs.RewriteConstants(map[string]interface{}{ - "the_constant": uint32(0x42424242), - }) - qt.Assert(t, qt.IsNil(err)) - qt.Assert(t, qt.ContentEquals(cs.Maps[".rodata"].Contents[0].Value.([]byte), []byte{0x42, 0x42, 0x42, 0x42})) -} - func TestCollectionSpec_LoadAndAssign_LazyLoading(t *testing.T) { spec := &CollectionSpec{ Maps: map[string]*MapSpec{ diff --git a/elf_reader.go b/elf_reader.go index cf7a31d43..75bbc506a 100644 --- a/elf_reader.go +++ b/elf_reader.go @@ -42,6 +42,7 @@ type elfCode struct { btf *btf.Spec extInfo *btf.ExtInfos maps map[string]*MapSpec + vars map[string]*VariableSpec kfuncs map[string]*btf.Func kconfig *MapSpec } @@ -100,7 +101,7 @@ func LoadCollectionSpecFromReader(rd io.ReaderAt) (*CollectionSpec, error) { sections[idx] = newElfSection(sec, mapSection) case sec.Name == ".maps": sections[idx] = newElfSection(sec, btfMapSection) - case sec.Name == ".bss" || strings.HasPrefix(sec.Name, ".data") || strings.HasPrefix(sec.Name, ".rodata"): + case isDataSection(sec.Name): sections[idx] = newElfSection(sec, dataSection) case sec.Type == elf.SHT_REL: // Store relocations under the section index of the target @@ -133,6 +134,7 @@ func LoadCollectionSpecFromReader(rd io.ReaderAt) (*CollectionSpec, error) { btf: btfSpec, extInfo: btfExtInfo, maps: make(map[string]*MapSpec), + vars: make(map[string]*VariableSpec), kfuncs: make(map[string]*btf.Func), } @@ -173,7 +175,7 @@ func LoadCollectionSpecFromReader(rd io.ReaderAt) (*CollectionSpec, error) { return nil, fmt.Errorf("load programs: %w", err) } - return &CollectionSpec{ec.maps, progs, btfSpec, ec.ByteOrder}, nil + return &CollectionSpec{ec.maps, progs, ec.vars, btfSpec, ec.ByteOrder}, nil } func loadLicense(sec *elf.Section) (string, error) { @@ -200,6 +202,10 @@ func loadVersion(sec *elf.Section, bo binary.ByteOrder) (uint32, error) { return version, nil } +func isDataSection(name string) bool { + return name == ".bss" || strings.HasPrefix(name, ".data") || strings.HasPrefix(name, ".rodata") +} + func isConstantDataSection(name string) bool { return strings.HasPrefix(name, ".rodata") } @@ -1101,6 +1107,10 @@ func (ec *elfCode) loadDataSections() error { continue } + if sec.Size > math.MaxUint32 { + return fmt.Errorf("data section %s: contents exceed maximum size", sec.Name) + } + mapSpec := &MapSpec{ Name: SanitizeName(sec.Name, -1), Type: Array, @@ -1116,20 +1126,47 @@ func (ec *elfCode) loadDataSections() error { if err != nil { return fmt.Errorf("data section %s: can't get contents: %w", sec.Name, err) } - - if uint64(len(data)) > math.MaxUint32 { - return fmt.Errorf("data section %s: contents exceed maximum size", sec.Name) - } mapSpec.Contents = []MapKV{{uint32(0), data}} case elf.SHT_NOBITS: - // NOBITS sections like .bss contain only zeroes, and since data sections - // are Arrays, the kernel already preallocates them. Skip reading zeroes - // from the ELF. + // NOBITS sections like .bss contain only zeroes and are not allocated in + // the ELF. Since data sections are Arrays, the kernel can preallocate + // them. Don't attempt reading zeroes from the ELF, instead allocate the + // zeroed memory to support getting and setting VariableSpecs for sections + // like .bss. + mapSpec.Contents = []MapKV{{uint32(0), make([]byte, sec.Size)}} + default: return fmt.Errorf("data section %s: unknown section type %s", sec.Name, sec.Type) } + for off, sym := range sec.symbols { + // Skip symbols marked with the 'hidden' attribute. + if elf.ST_VISIBILITY(sym.Other) == elf.STV_HIDDEN { + continue + } + + if ec.vars[sym.Name] != nil { + return fmt.Errorf("data section %s: duplicate variable %s", sec.Name, sym.Name) + } + + // Skip symbols starting with a dot, they are compiler-internal symbols + // emitted by clang 11 and earlier and are not cleaned up by the bpf + // compiler backend (e.g. symbols named .Lconstinit.1 in sections like + // .rodata.cst32). Variables in C cannot start with a dot, so filter these + // out. + if strings.HasPrefix(sym.Name, ".") { + continue + } + + ec.vars[sym.Name] = &VariableSpec{ + name: sym.Name, + offset: off, + size: sym.Size, + m: mapSpec, + } + } + // It is possible for a data section to exist without a corresponding BTF Datasec // if it only contains anonymous values like macro-defined arrays. if ec.btf != nil { @@ -1138,6 +1175,34 @@ func (ec *elfCode) loadDataSections() error { // Assign the spec's key and BTF only if the Datasec lookup was successful. mapSpec.Key = &btf.Void{} mapSpec.Value = ds + + // Populate VariableSpecs with type information, if available. + for _, v := range ds.Vars { + name := v.Type.TypeName() + if name == "" { + return fmt.Errorf("data section %s: anonymous variable %v", sec.Name, v) + } + + if _, ok := v.Type.(*btf.Var); !ok { + return fmt.Errorf("data section %s: unexpected type %T for variable %s", sec.Name, v.Type, name) + } + + ev := ec.vars[name] + if ev == nil { + // Hidden symbols appear in the BTF Datasec but don't receive a VariableSpec. + continue + } + + if uint64(v.Offset) != ev.offset { + return fmt.Errorf("data section %s: variable %s datasec offset (%d) doesn't match ELF symbol offset (%d)", sec.Name, name, v.Offset, ev.offset) + } + + if uint64(v.Size) != ev.size { + return fmt.Errorf("data section %s: variable %s size in datasec (%d) doesn't match ELF symbol size (%d)", sec.Name, name, v.Size, ev.size) + } + + ev.t = v.Type + } } } diff --git a/elf_reader_test.go b/elf_reader_test.go index 089c4578a..702803561 100644 --- a/elf_reader_test.go +++ b/elf_reader_test.go @@ -96,13 +96,64 @@ func TestLoadCollectionSpec(t *testing.T) { ValueSize: 8, MaxEntries: 1, }, - ".data.custom": { - Name: SanitizeName(".data.custom", -1), + ".bss": { + Name: SanitizeName(".bss", -1), Type: Array, KeySize: 4, ValueSize: 4, MaxEntries: 1, - Contents: []MapKV{{Key: uint32(0), Value: make([]uint8, 4)}}, + }, + ".data": { + Name: SanitizeName(".data", -1), + Type: Array, + KeySize: 4, + ValueSize: 4, + MaxEntries: 1, + }, + ".data.test": { + Name: SanitizeName(".data.test", -1), + Type: Array, + KeySize: 4, + ValueSize: 4, + MaxEntries: 1, + }, + ".data.hidden": { + Name: SanitizeName(".data.hidden", -1), + Type: Array, + KeySize: 4, + ValueSize: 4, + MaxEntries: 1, + }, + ".data.struct": { + Name: SanitizeName(".data.struct", -1), + Type: Array, + KeySize: 4, + ValueSize: 16, + MaxEntries: 1, + }, + ".rodata": { + Name: SanitizeName(".rodata", -1), + Type: Array, + KeySize: 4, + ValueSize: 24, + MaxEntries: 1, + Flags: sys.BPF_F_RDONLY_PROG, + }, + ".rodata.test": { + Name: SanitizeName(".rodata.test", -1), + Type: Array, + KeySize: 4, + ValueSize: 4, + MaxEntries: 1, + Flags: sys.BPF_F_RDONLY_PROG, + }, + ".rodata.cst32": { + Name: SanitizeName(".rodata.cst32", -1), + Type: Array, + KeySize: 4, + ValueSize: 32, + MaxEntries: 1, + Flags: sys.BPF_F_RDONLY_PROG, }, }, Programs: map[string]*ProgramSpec{ @@ -148,6 +199,25 @@ func TestLoadCollectionSpec(t *testing.T) { SectionName: "socket/4", License: "MIT", }, + "set_vars": { + Name: "set_vars", + Type: SocketFilter, + SectionName: "socket", + License: "MIT", + }, + }, + Variables: map[string]*VariableSpec{ + "arg": {name: "arg", offset: 4, size: 4}, + "arg2": {name: "arg2", offset: 0, size: 4}, + "arg3": {name: "arg3", offset: 0, size: 4}, + "key1": {name: "key1", offset: 0, size: 4}, + "key2": {name: "key2", offset: 0, size: 4}, + "key3": {name: "key3", offset: 0, size: 4}, + "neg": {name: "neg", offset: 12, size: 4}, + "static_neg": {name: "static_neg", offset: 20, size: 4}, + "static_uneg": {name: "static_uneg", offset: 16, size: 4}, + "struct_var": {name: "struct_var", offset: 0, size: 16}, + "uneg": {name: "uneg", offset: 8, size: 4}, }, } @@ -159,17 +229,17 @@ func TestLoadCollectionSpec(t *testing.T) { } return false }), + cmp.Comparer(func(a, b *VariableSpec) bool { + if a.name != b.name || a.offset != b.offset || a.size != b.size { + return false + } + return true + }), cmpopts.IgnoreTypes(new(btf.Spec)), cmpopts.IgnoreFields(CollectionSpec{}, "ByteOrder", "Types"), cmpopts.IgnoreFields(ProgramSpec{}, "Instructions", "ByteOrder"), - cmpopts.IgnoreFields(MapSpec{}, "Key", "Value"), + cmpopts.IgnoreFields(MapSpec{}, "Key", "Value", "Contents"), cmpopts.IgnoreUnexported(ProgramSpec{}), - cmpopts.IgnoreMapEntries(func(key string, _ *MapSpec) bool { - if key == ".bss" || key == ".data" || strings.HasPrefix(key, ".rodata") { - return true - } - return false - }), } testutils.Files(t, testutils.Glob(t, "testdata/loader-*.elf"), func(t *testing.T, file string) { @@ -187,12 +257,19 @@ func TestLoadCollectionSpec(t *testing.T) { } err = have.RewriteConstants(map[string]interface{}{ - "totallyBogus": uint32(1), + "totallyBogus": uint32(1), + "totallyBogus2": uint32(2), }) if err == nil { t.Error("Rewriting a bogus constant doesn't fail") } + var mErr *MissingConstantsError + if !errors.As(err, &mErr) { + t.Fatal("Error doesn't wrap MissingConstantsError:", err) + } + qt.Assert(t, qt.ContentEquals(mErr.Constants, []string{"totallyBogus", "totallyBogus2"})) + if diff := cmp.Diff(coll, have, cmpOpts...); diff != "" { t.Errorf("MapSpec mismatch (-want +got):\n%s", diff) } diff --git a/go.mod b/go.mod index aca8492f9..d20a71270 100644 --- a/go.mod +++ b/go.mod @@ -6,16 +6,21 @@ require ( github.com/go-quicktest/qt v1.101.0 github.com/google/go-cmp v0.6.0 github.com/jsimonetti/rtnetlink/v2 v2.0.1 + github.com/stretchr/testify v1.9.0 golang.org/x/sys v0.20.0 + golang.org/x/text v0.14.0 ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/josharian/native v1.1.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mdlayher/netlink v1.7.2 // indirect github.com/mdlayher/socket v0.4.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sync v0.1.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e0fee7777..33f298874 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -16,12 +18,22 @@ github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/sysenc/buffer.go b/internal/sysenc/buffer.go index d184ea196..070a21126 100644 --- a/internal/sysenc/buffer.go +++ b/internal/sysenc/buffer.go @@ -51,12 +51,12 @@ func SyscallOutput(dst any, size int) Buffer { // // Returns the number of copied bytes. func (b Buffer) CopyTo(dst []byte) int { - return copy(dst, b.unsafeBytes()) + return copy(dst, b.Bytes()) } // AppendTo appends the buffer onto dst. func (b Buffer) AppendTo(dst []byte) []byte { - return append(dst, b.unsafeBytes()...) + return append(dst, b.Bytes()...) } // Pointer returns the location where a syscall should write. @@ -72,10 +72,10 @@ func (b Buffer) Unmarshal(data any) error { return nil } - return Unmarshal(data, b.unsafeBytes()) + return Unmarshal(data, b.Bytes()) } -func (b Buffer) unsafeBytes() []byte { +func (b Buffer) Bytes() []byte { if b.size == syscallPointerOnly { return nil } diff --git a/map.go b/map.go index 36fe27418..43762598d 100644 --- a/map.go +++ b/map.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "slices" "strings" "sync" @@ -191,6 +192,10 @@ func (ms *MapSpec) readOnly() bool { return (ms.Flags & sys.BPF_F_RDONLY_PROG) > 0 } +func (ms *MapSpec) writeOnly() bool { + return (ms.Flags & sys.BPF_F_WRONLY_PROG) > 0 +} + // MapKV is used to initialize the contents of a Map. type MapKV struct { Key interface{} @@ -255,6 +260,8 @@ type Map struct { pinnedPath string // Per CPU maps return values larger than the size in the spec fullValueSize int + + memory *Memory } // NewMapFromFD creates a map from a raw fd. @@ -388,6 +395,44 @@ func newMapWithOptions(spec *MapSpec, opts MapOptions) (_ *Map, err error) { return m, nil } +// Memory returns a memory-mapped region for the Map. Operations are +// concurrency-safe since the object doesn't maintain any state. Repeated calls +// to Memory return the same mapping. +// +// Callers are responsible for coordinating access to the resulting Memory. +func (m *Map) Memory() (*Memory, error) { + if m.memory != nil { + return m.memory, nil + } + + if m.flags&unix.BPF_F_MMAPABLE == 0 { + return nil, fmt.Errorf("Map was not created with the BPF_F_MMAPABLE flag") + } + + var ro bool + flags := unix.PROT_READ | unix.PROT_WRITE + if m.flags&unix.BPF_F_RDONLY_PROG > 0 { + ro = true + flags = unix.PROT_READ + } + + //TODO: Size calc is different for arena maps, add helper to *Map. + b, err := unix.Mmap(m.FD(), 0, int(m.ValueSize()*m.MaxEntries()), flags, unix.MAP_SHARED) + if err != nil { + return nil, fmt.Errorf("setting up memory-mapped region: %w", err) + } + + mm := &Memory{ + b, + ro, + } + runtime.SetFinalizer(mm, (*Memory).close) + + m.memory = mm + + return mm, nil +} + // createMap validates the spec's properties and creates the map in the kernel // using the given opts. It does not populate or freeze the map. func (spec *MapSpec) createMap(inner *sys.FD) (_ *Map, err error) { @@ -412,6 +457,11 @@ func (spec *MapSpec) createMap(inner *sys.FD) (_ *Map, err error) { return nil, err } + // TODO: Do this properly. + if haveMmapableMaps() == nil && isDataSection(spec.Name) { + spec.Flags |= sys.BPF_F_MMAPABLE + } + attr := sys.MapCreateAttr{ MapType: sys.MapType(spec.Type), KeySize: spec.KeySize, @@ -479,13 +529,13 @@ func handleMapCreateError(attr sys.MapCreateAttr, spec *MapSpec, err error) erro return fmt.Errorf("map create: %w (noPrealloc flag may be incompatible with map type %s)", err, spec.Type) } - switch spec.Type { - case ArrayOfMaps, HashOfMaps: + if spec.Type.canStoreMap() { if haveFeatErr := haveNestedMaps(); haveFeatErr != nil { return fmt.Errorf("map create: %w", haveFeatErr) } } - if spec.Flags&(sys.BPF_F_RDONLY_PROG|sys.BPF_F_WRONLY_PROG) > 0 { + + if spec.readOnly() || spec.writeOnly() { if haveFeatErr := haveMapMutabilityModifiers(); haveFeatErr != nil { return fmt.Errorf("map create: %w", haveFeatErr) } @@ -531,6 +581,7 @@ func newMap(fd *sys.FD, name string, typ MapType, keySize, valueSize, maxEntries flags, "", int(valueSize), + nil, } if !typ.hasPerCPUValue() { @@ -1337,6 +1388,7 @@ func (m *Map) Clone() (*Map, error) { m.flags, "", m.fullValueSize, + nil, }, nil } diff --git a/map_mmap.go b/map_mmap.go new file mode 100644 index 000000000..fb2f60ca2 --- /dev/null +++ b/map_mmap.go @@ -0,0 +1,165 @@ +package ebpf + +import ( + "fmt" + "io" + "sync/atomic" + "unsafe" + + "github.com/cilium/ebpf/internal/unix" +) + +type Memory struct { + // Pointer to the memory-mapped region. + b []byte + ro bool +} + +func (mm *Memory) Size() int { + return len(mm.b) +} + +func (mm *Memory) close() error { + if err := unix.Munmap(mm.b); err != nil { + return fmt.Errorf("unmapping memory-mapped region: %w", err) + } + + mm.b = nil + + return nil +} + +func (mm *Memory) ReadAt(p []byte, off int64) (int, error) { + if mm.b == nil { + return 0, fmt.Errorf("memory-mapped region closed") + } + + if p == nil { + return 0, fmt.Errorf("input buffer p is nil") + } + + if off < 0 || off >= int64(len(mm.b)) { + return 0, fmt.Errorf("read offset out of range") + } + + n := copy(p, mm.b[off:]) + if n < len(p) { + return n, io.EOF + } + + return n, nil +} + +func (mm *Memory) WriteAt(p []byte, off int64) (int, error) { + if mm.b == nil { + return 0, fmt.Errorf("memory-mapped region closed") + } + if mm.ro { + return 0, fmt.Errorf("memory-mapped region is read-only") + } + + if p == nil { + return 0, fmt.Errorf("output buffer p is nil") + } + + if off < 0 || off >= int64(len(mm.b)) { + return 0, fmt.Errorf("write offset out of range") + } + + n := copy(mm.b[off:], p) + if n < len(p) { + return n, io.EOF + } + + return n, nil +} + +// Uint32 provides atomic access to a uint32 in a memory-mapped region. +type Uint32 struct { + *atomic.Uint32 + mm *Memory +} + +// Uint64 provides atomic access to a uint64 in a memory-mapped region. +type Uint64 struct { + *atomic.Uint64 + mm *Memory +} + +// Int32 provides atomic access to an int32 in a memory-mapped region. +type Int32 struct { + *atomic.Int32 + mm *Memory +} + +// Int64 provides atomic access to an int64 in a memory-mapped region. +type Int64 struct { + *atomic.Int64 + mm *Memory +} + +// checkMemory ensures a T can be accessed in mm at offset off. Returns an error +// if mm is read-only. +func checkMemory[T any](mm *Memory, off uint64) error { + var t T + if mm.b == nil { + return fmt.Errorf("memory-mapped region closed") + } + if mm.ro { + return fmt.Errorf("memory-mapped region is read-only") + } + vs, bs := uint64(unsafe.Sizeof(t)), uint64(len(mm.b)) + if off+vs > bs { + return fmt.Errorf("%d-byte write at offset %d exceeds mmap size of %d bytes", vs, off, bs) + } + return nil +} + +// reinterp reinterprets a pointer of type In to a pointer of type Out. +func reinterp[Out, In any](in *In) *Out { + return (*Out)(unsafe.Pointer(in)) +} + +// AtomicUint32 returns an atomic accessor to a uint32 in the memory-mapped +// region at offset off. +// +// It's not possible to obtain an accessor for a read-only region. +func (mm *Memory) AtomicUint32(off uint64) (r *Uint32, err error) { + if err := checkMemory[atomic.Uint32](mm, off); err != nil { + return nil, err + } + return &Uint32{reinterp[atomic.Uint32](&mm.b[off]), mm}, nil +} + +// AtomicInt32 returns an atomic accessor to an int32 in the memory-mapped +// region at offset off. +// +// It's not possible to obtain an accessor for a read-only region. +func (mm *Memory) AtomicInt32(off uint64) (r *Int32, err error) { + if err := checkMemory[atomic.Int32](mm, off); err != nil { + return nil, err + } + return &Int32{reinterp[atomic.Int32](&mm.b[off]), mm}, nil +} + +// AtomicUint64 returns an atomic accessor to a uint64 in the memory-mapped +// region at offset off. +// +// It's not possible to obtain an accessor for a read-only region. +func (mm *Memory) AtomicUint64(off uint64) (r *Uint64, err error) { + if err := checkMemory[atomic.Uint64](mm, off); err != nil { + return nil, err + } + return &Uint64{reinterp[atomic.Uint64](&mm.b[off]), mm}, nil +} + +// AtomicInt64 returns an atomic accessor to an int64 in the memory-mapped +// region at offset off. +// +// It's not possible to obtain an accessor for a read-only region. +func (mm *Memory) AtomicInt64(off uint64) (r *Int64, err error) { + if err := checkMemory[atomic.Int64](mm, off); err != nil { + return nil, err + } + return &Int64{reinterp[atomic.Int64](&mm.b[off]), mm}, nil +} diff --git a/map_mmap_test.go b/map_mmap_test.go new file mode 100644 index 000000000..ec262330d --- /dev/null +++ b/map_mmap_test.go @@ -0,0 +1,66 @@ +package ebpf + +import ( + "encoding/hex" + "fmt" + "io" + "testing" + + "github.com/cilium/ebpf/internal/sys" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMmap(t *testing.T) { + m, err := NewMap(&MapSpec{ + Name: "mmap", + Type: Array, + KeySize: 4, + ValueSize: 8, + MaxEntries: 2, + Contents: []MapKV{ + {Key: uint32(0), Value: []byte{0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00}}, + {Key: uint32(1), Value: []byte{0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00}}, + }, + Flags: sys.BPF_F_MMAPABLE, + }) + require.NoError(t, err) + defer m.Close() + + mm, err := m.Memory() + require.NoError(t, err) + defer mm.close() + + // write + w := io.NewOffsetWriter(mm, 10) + n, err := w.Write([]byte{1, 2, 3, 4}) + require.NoError(t, err) + assert.Equal(t, n, 4) + + // atomics + u32, err := mm.AtomicUint32(12) + require.NoError(t, err) + u32.Store(0xbeef) + + fmt.Println("mmaped region:", mm.b) + + // atomic read + require.NoError(t, err) + assert.Equal(t, u32.Load(), uint32(0xbeef)) + + // atomic add + u32.Add(1) + require.NoError(t, err) + assert.EqualValues(t, u32.Load(), 0xbeef+1) + + // read + r := io.NewSectionReader(mm, 10, 6) + buf := make([]byte, 6) + n, err = r.Read(buf) + require.NoError(t, err) + assert.Equal(t, n, 6) + fmt.Println("read:", buf) + + fmt.Println("dump:", hex.Dump(mm.b)) +} diff --git a/testdata/common.h b/testdata/common.h index 0a31d1324..00b7c4c18 100644 --- a/testdata/common.h +++ b/testdata/common.h @@ -24,6 +24,8 @@ enum libbpf_tristate { #define __weak __attribute__((weak)) #endif +#define __hidden __attribute__((visibility("hidden"))) + #define bpf_ksym_exists(sym) \ ({ \ _Static_assert(!__builtin_constant_p(!!sym), #sym " should be marked as __weak"); \ diff --git a/testdata/loader-clang-11-eb.elf b/testdata/loader-clang-11-eb.elf index 11ba1a4d1..b0be7db88 100644 Binary files a/testdata/loader-clang-11-eb.elf and b/testdata/loader-clang-11-eb.elf differ diff --git a/testdata/loader-clang-11-el.elf b/testdata/loader-clang-11-el.elf index 39dcab855..fb2875156 100644 Binary files a/testdata/loader-clang-11-el.elf and b/testdata/loader-clang-11-el.elf differ diff --git a/testdata/loader-clang-14-eb.elf b/testdata/loader-clang-14-eb.elf index db08f3cc5..4814acbac 100644 Binary files a/testdata/loader-clang-14-eb.elf and b/testdata/loader-clang-14-eb.elf differ diff --git a/testdata/loader-clang-14-el.elf b/testdata/loader-clang-14-el.elf index d8cb5f684..21535596d 100644 Binary files a/testdata/loader-clang-14-el.elf and b/testdata/loader-clang-14-el.elf differ diff --git a/testdata/loader-clang-17-eb.elf b/testdata/loader-clang-17-eb.elf index db08f3cc5..4814acbac 100644 Binary files a/testdata/loader-clang-17-eb.elf and b/testdata/loader-clang-17-eb.elf differ diff --git a/testdata/loader-clang-17-el.elf b/testdata/loader-clang-17-el.elf index d8cb5f684..21535596d 100644 Binary files a/testdata/loader-clang-17-el.elf and b/testdata/loader-clang-17-el.elf differ diff --git a/testdata/loader.c b/testdata/loader.c index bb898f067..b9a0ba6b8 100644 --- a/testdata/loader.c +++ b/testdata/loader.c @@ -102,7 +102,7 @@ static volatile const uint32_t arg; // .rodata, populated by loader // custom .rodata section, populated by loader static volatile const uint32_t arg2 __section(".rodata.test"); // custom .data section -static volatile uint32_t arg3 __section(".data.custom"); +static volatile uint32_t arg3 __section(".data.test"); __section("xdp") int xdp_prog() { map_lookup_elem(&hash_map, (void *)&key1); @@ -144,6 +144,22 @@ __section("socket/3") int data_sections() { return 0; } +// Should not appear in CollectionSpec.Variables. +__hidden volatile uint32_t hidden __section(".data.hidden"); + +struct struct_var_t { + uint64_t a; + uint64_t b; +}; +static volatile struct struct_var_t struct_var __section(".data.struct"); + +__section("socket") int set_vars() { + hidden = 0xbeef1; + struct_var.a = 0xbeef2; + struct_var.b = 0xbeef3; + return 0; +} + /* * Up until LLVM 14, this program results in an .rodata.cst32 section * that is accessed by 'return values[i]'. For this section, no BTF is diff --git a/variable.go b/variable.go new file mode 100644 index 000000000..71733a7c7 --- /dev/null +++ b/variable.go @@ -0,0 +1,245 @@ +package ebpf + +import ( + "fmt" + "sync/atomic" + "unsafe" + + "github.com/cilium/ebpf/btf" + "github.com/cilium/ebpf/internal/sysenc" +) + +// VariableSpec is a convenience wrapper for modifying global variables of a +// CollectionSpec before loading it into the kernel. +// +// All operations on a VariableSpec's underlying MapSpec are performed in the +// host's native endianness. +type VariableSpec struct { + name string + offset uint64 + size uint64 + + m *MapSpec + t btf.Type +} + +// Set sets the value of the VariableSpec to the provided input using the host's +// native endianness. +func (s *VariableSpec) Set(in any) error { + buf, err := sysenc.Marshal(in, int(s.size)) + if err != nil { + return fmt.Errorf("marshaling value %s: %w", s.name, err) + } + + b, _, err := s.m.dataSection() + if err != nil { + return fmt.Errorf("getting data section of map %s: %w", s.m.Name, err) + } + + if int(s.offset+s.size) > len(b) { + return fmt.Errorf("offset %d(+%d) for variable %s is out of bounds", s.offset, s.size, s.name) + } + + // MapSpec.Copy() performs a shallow copy. Fully copy the byte slice + // to avoid any changes affecting other copies of the MapSpec. + cpy := make([]byte, len(b)) + copy(cpy, b) + + buf.CopyTo(cpy[s.offset : s.offset+s.size]) + + s.m.Contents[0] = MapKV{Key: uint32(0), Value: cpy} + + return nil +} + +// Get writes the value of the VariableSpec to the provided output using the +// host's native endianness. +func (s *VariableSpec) Get(out any) error { + b, _, err := s.m.dataSection() + if err != nil { + return fmt.Errorf("getting data section of map %s: %w", s.m.Name, err) + } + + if int(s.offset+s.size) > len(b) { + return fmt.Errorf("offset %d(+%d) for variable %s is out of bounds", s.offset, s.size, s.name) + } + + if err := sysenc.Unmarshal(out, b[s.offset:s.offset+s.size]); err != nil { + return fmt.Errorf("unmarshaling value: %w", err) + } + + return nil +} + +// Size returns the size of the VariableSpec in bytes. +func (s *VariableSpec) Size() uint64 { + return s.size +} + +// Constant returns true if the VariableSpec represents a variable that is +// read-only from the perspective of the bpf program. +func (s *VariableSpec) Constant() bool { + return s.m.readOnly() +} + +// Type returns the BTF type of the variable. It contains the [btf.Var] wrapping +// the underlying variable's type. +func (s *VariableSpec) Type() btf.Type { + return s.t +} + +func (s *VariableSpec) String() string { + return fmt.Sprintf("%s (type=%v, map=%s, offset=%d, size=%d)", s.name, s.t, s.m.Name, s.offset, s.size) +} + +// copy returns a new VariableSpec with the same values as the original, +// but with a different underlying MapSpec. This is useful when copying a +// CollectionSpec. Returns nil if a MapSpec with the same name is not found. +func (s *VariableSpec) copy(cpy *CollectionSpec) *VariableSpec { + out := &VariableSpec{ + name: s.name, + offset: s.offset, + size: s.size, + t: s.t, + } + + // Attempt to find a MapSpec with the same name in the copied CollectionSpec. + for _, m := range cpy.Maps { + if m.Name == s.m.Name { + out.m = m + return out + } + } + + return nil +} + +// Variable is a convenience wrapper for modifying global variables of a +// Collection after loading it into the kernel. +// +// Operations on a Variable's underlying Map are performed in the host's native +// endianness and using direct memory access, bypassing the BPF map syscall API. +// As such, Variables are only supported on Linux 5.5 and later or on kernels +// supporting BPF_F_MMAPABLE. +type Variable struct { + name string + offset uint64 + size uint64 + ro bool + + mm *Memory + t btf.Type +} + +// Size returns the size of the variable. +func (v *Variable) Size() uint64 { + return v.size +} + +// Type returns the BTF type of the variable. It contains the [btf.Var] wrapping +// the underlying variable's type. +func (v *Variable) Type() btf.Type { + return v.t +} + +func (v *Variable) String() string { + return fmt.Sprintf("%s (type=%v)", v.name, v.t) +} + +// Set the value of the Variable to the provided input. The input must marshal +// to the same length as the size of the Variable. +func (v *Variable) Set(in any) error { + if v.ro { + return fmt.Errorf("variable %s is read-only", v.name) + } + + buf, err := sysenc.Marshal(in, int(v.size)) + if err != nil { + return fmt.Errorf("marshaling value %s: %w", v.name, err) + } + + if int(v.offset+v.size) > v.mm.Size() { + return fmt.Errorf("offset %d(+%d) for variable %s is out of bounds", v.offset, v.size, v.name) + } + + if _, err := v.mm.WriteAt(buf.Bytes(), int64(v.offset)); err != nil { + return fmt.Errorf("writing value to %s: %w", v.name, err) + } + + return nil +} + +// Get writes the value of the Variable to the provided output. The output must +// be a pointer to a value whose size matches the Variable. +func (v *Variable) Get(out any) error { + if int(v.offset+v.size) > v.mm.Size() { + return fmt.Errorf("offset %d(+%d) for variable %s is out of bounds", v.offset, v.size, v.name) + } + + b := make([]byte, v.size) + if _, err := v.mm.ReadAt(b, int64(v.offset)); err != nil { + return fmt.Errorf("reading value from %s: %w", v.name, err) + } + + if err := sysenc.Unmarshal(out, b); err != nil { + return fmt.Errorf("unmarshaling value: %w", err) + } + + return nil +} + +func checkAtomic[T any](v *Variable) error { + var t T + if v.ro { + return fmt.Errorf("variable %s is read-only", v.name) + } + + if v.size != uint64(unsafe.Sizeof(t)) { + return fmt.Errorf("variable %s is not %d bytes", v.name, v.size) + } + return nil +} + +// AtomicUint32 returns an atomic accessor to a uint32 Variable. Only valid for +// Variables that are 32 bits in size. +// +// It's not possible to obtain an accessor for a constant Variable. +func (v *Variable) AtomicUint32() (*Uint32, error) { + if err := checkAtomic[atomic.Uint32](v); err != nil { + return nil, err + } + return v.mm.AtomicUint32(v.offset) +} + +// AtomicInt32 returns an atomic accessor to an int32 Variable. Only valid for +// Variables that are 32 bits in size. +// +// It's not possible to obtain an accessor for a constant Variable. +func (v *Variable) AtomicInt32() (*Int32, error) { + if err := checkAtomic[atomic.Int32](v); err != nil { + return nil, err + } + return v.mm.AtomicInt32(v.offset) +} + +// AtomicUint64 returns an atomic accessor to a uint64 Variable. Only valid for +// Variables that are 64 bits in size. +// +// It's not possible to obtain an accessor for a constant Variable. +func (v *Variable) AtomicUint64() (*Uint64, error) { + if err := checkAtomic[atomic.Uint64](v); err != nil { + return nil, err + } + return v.mm.AtomicUint64(v.offset) +} + +// AtomicInt64 returns an atomic accessor to an int64 Variable. Only valid for +// Variables that are 64 bits in size. +// +// It's not possible to obtain an accessor for a constant Variable. +func (v *Variable) AtomicInt64() (*Int64, error) { + if err := checkAtomic[atomic.Int64](v); err != nil { + return nil, err + } + return v.mm.AtomicInt64(v.offset) +} diff --git a/variable_test.go b/variable_test.go new file mode 100644 index 000000000..8af0242f2 --- /dev/null +++ b/variable_test.go @@ -0,0 +1,64 @@ +package ebpf + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/cilium/ebpf/internal/testutils" +) + +func TestVariableSpec(t *testing.T) { + file := testutils.NativeFile(t, "testdata/loader-%s.elf") + spec, err := LoadCollectionSpec(file) + if err != nil { + t.Fatal(err) + } + + assert.Nil(t, spec.Variables["hidden"]) + + const want uint32 = 12345 + + // Update a variable in each type of data section (.bss,.data,.rodata) + assert.NoError(t, spec.Variables["key1"].Set(want)) + assert.NoError(t, spec.Variables["key2"].Set(want)) + assert.NoError(t, spec.Variables["key3"].Set(want)) + + var v uint32 + assert.NoError(t, spec.Variables["key1"].Get(&v)) + assert.EqualValues(t, want, v) + assert.NoError(t, spec.Variables["key2"].Get(&v)) + assert.EqualValues(t, want, v) + assert.NoError(t, spec.Variables["key3"].Get(&v)) + assert.EqualValues(t, want, v) + + // Composite values. + type structT struct { + A, B uint64 + } + assert.NoError(t, spec.Variables["struct_var"].Set(structT{1, 2})) + + var s structT + assert.NoError(t, spec.Variables["struct_var"].Get(&s)) + assert.Equal(t, structT{1, 2}, s) +} + +func TestVariableSpecCopy(t *testing.T) { + file := testutils.NativeFile(t, "testdata/loader-%s.elf") + spec, err := LoadCollectionSpec(file) + if err != nil { + t.Fatal(err) + } + + cpy := spec.Copy() + + // Update a variable in a section with only a single variable (.rodata.test). + const want uint32 = 0xfefefefe + wantb := []byte{0xfe, 0xfe, 0xfe, 0xfe} // Same byte sequence regardless of endianness + assert.NoError(t, cpy.Variables["arg2"].Set(want)) + assert.Equal(t, wantb, cpy.Maps[".rodata.test"].Contents[0].Value) + + // Verify that the original underlying MapSpec was not modified. + zero := make([]byte, 4) + assert.Equal(t, zero, spec.Maps[".rodata.test"].Contents[0].Value) +}