feat(userland): add support for policy versioning

Add policy versioning, which will support runtime policy updates.

This also:

- Adds the singleton Snapshots circular buffer (for policy versioning).
- Introduces the Cloner interface (used for all policy objects).
- Moves the filters' ebpf logic to a new ebpf file (filter types are now
  ebpf-agnostic).
- Splits filters_test.go into multiple files.
- Removes the unused containers filter (it was just a wrapper for
  StringFilter).
- Removes unused RandStringRunes function from sets_test.go.
- Makes other minor refactorings/changes.

common/interfaces.go

@@ -0,0 +1,6 @@
+package common
+
+// Cloner is an interface for objects that can be cloned
+type Cloner interface {
+	Clone() Cloner
+}

pkg/bufferdecoder/decoder.go

@@ -75,11 +75,11 @@ func (decoder *EbpfDecoder) DecodeContext(eCtx *EventContext) error {
 
 	eCtx.EventID = events.ID(int32(binary.LittleEndian.Uint32(decoder.buffer[offset+112 : offset+116])))
 	eCtx.Syscall = int32(binary.LittleEndian.Uint32(decoder.buffer[offset+116 : offset+120]))
-	eCtx.MatchedPolicies = binary.LittleEndian.Uint64(decoder.buffer[offset+120 : offset+128])
-	eCtx.Retval = int64(binary.LittleEndian.Uint64(decoder.buffer[offset+128 : offset+136]))
-	eCtx.StackID = binary.LittleEndian.Uint32(decoder.buffer[offset+136 : offset+140])
-	eCtx.ProcessorId = binary.LittleEndian.Uint16(decoder.buffer[offset+140 : offset+142])
-	// 2 byte padding
+	eCtx.Retval = int64(binary.LittleEndian.Uint64(decoder.buffer[offset+120 : offset+128]))
+	eCtx.StackID = binary.LittleEndian.Uint32(decoder.buffer[offset+128 : offset+132])
+	eCtx.ProcessorId = binary.LittleEndian.Uint16(decoder.buffer[offset+132 : offset+134])
+	eCtx.PoliciesVersion = binary.LittleEndian.Uint16(decoder.buffer[offset+134 : offset+136])
+	eCtx.MatchedPolicies = binary.LittleEndian.Uint64(decoder.buffer[offset+136 : offset+144])
 	// event_context end
 
 	decoder.cursor += eCtx.GetSizeBytes()

pkg/bufferdecoder/decoder_test.go

@@ -15,23 +15,30 @@ func TestDecodeContext(t *testing.T) {
 
 	buf := new(bytes.Buffer)
 	eCtxExpected := EventContext{
-		Ts:          11,
-		CgroupID:    22,
-		ProcessorId: 5,
-		Pid:         543,
-		Tid:         77,
-		Ppid:        4567,
-		HostPid:     5430,
-		HostTid:     124,
-		HostPpid:    555,
-		Uid:         9876,
-		MntID:       1357,
-		PidID:       3758,
-		Comm:        [16]byte{1, 3, 5, 3, 1, 5, 56, 6, 7, 32, 2, 4},
-		UtsName:     [16]byte{5, 6, 7, 8, 9, 4, 3, 2},
-		EventID:     0,
-		Retval:      0,
-		StackID:     0,
+		Ts:              11,
+		StartTime:       0,
+		CgroupID:        22,
+		Pid:             543,
+		Tid:             77,
+		Ppid:            4567,
+		HostPid:         5430,
+		HostTid:         124,
+		HostPpid:        555,
+		Uid:             9876,
+		MntID:           1357,
+		PidID:           3758,
+		Comm:            [16]byte{1, 3, 5, 3, 1, 5, 56, 6, 7, 32, 2, 4},
+		UtsName:         [16]byte{5, 6, 7, 8, 9, 4, 3, 2},
+		Flags:           0,
+		LeaderStartTime: 1331,
+		ParentStartTime: 1221,
+		EventID:         5,
+		Syscall:         0,
+		Retval:          0,
+		StackID:         0,
+		ProcessorId:     5,
+		PoliciesVersion: 11,
+		MatchedPolicies: 1917,
 	}
 	err := binary.Write(buf, binary.LittleEndian, eCtxExpected)
 	assert.Equal(t, nil, err)

pkg/bufferdecoder/protocol.go

@@ -45,11 +45,11 @@ type EventContext struct {
 
 	EventID         events.ID // int32
 	Syscall         int32
-	MatchedPolicies uint64
 	Retval          int64
 	StackID         uint32
 	ProcessorId     uint16
-	_               [2]byte // padding
+	PoliciesVersion uint16
+	MatchedPolicies uint64
 }
 
 func (EventContext) GetSizeBytes() int {

pkg/cmd/cobra/cobra.go

@@ -232,6 +232,7 @@ func GetTraceeRunner(c *cobra.Command, version string) (cmd.Runner, error) {
 	}
 
 	cfg.Policies = policies
+	policy.Snapshots().Store(cfg.Policies)
 
 	// Output command line flags
 

pkg/cmd/urfave/urfave.go

@@ -13,6 +13,7 @@ import (
 	"github.com/aquasecurity/tracee/pkg/config"
 	"github.com/aquasecurity/tracee/pkg/errfmt"
 	"github.com/aquasecurity/tracee/pkg/logger"
+	"github.com/aquasecurity/tracee/pkg/policy"
 )
 
 func GetTraceeRunner(c *cli.Context, version string) (cmd.Runner, error) {
@@ -125,8 +126,8 @@ func GetTraceeRunner(c *cli.Context, version string) (cmd.Runner, error) {
 	if err != nil {
 		return runner, err
 	}
-
 	cfg.Policies = policies
+	policy.Snapshots().Store(cfg.Policies)
 
 	broadcast, err := printer.NewBroadcast(output.PrinterConfigs, cmd.GetContainerMode(cfg))
 	if err != nil {

pkg/ebpf/events_pipeline.go

@@ -12,6 +12,7 @@ import (
 	"github.com/aquasecurity/tracee/pkg/errfmt"
 	"github.com/aquasecurity/tracee/pkg/events"
 	"github.com/aquasecurity/tracee/pkg/logger"
+	"github.com/aquasecurity/tracee/pkg/policy"
 	"github.com/aquasecurity/tracee/pkg/utils"
 	"github.com/aquasecurity/tracee/types/trace"
 )
@@ -240,6 +241,7 @@ func (t *Tracee) decodeEvents(ctx context.Context, sourceChan chan []byte) (<-ch
 			evt.Kubernetes = kubernetesData
 			evt.EventID = int(eCtx.EventID)
 			evt.EventName = eventDefinition.GetName()
+			evt.PoliciesVersion = eCtx.PoliciesVersion
 			evt.MatchedPoliciesKernel = eCtx.MatchedPolicies
 			evt.MatchedPoliciesUser = 0
 			evt.MatchedPolicies = []string{}
@@ -288,13 +290,19 @@ func (t *Tracee) matchPolicies(event *trace.Event) uint64 {
 	eventID := events.ID(event.EventID)
 	bitmap := event.MatchedPoliciesKernel
 
+	policies, err := policy.Snapshots().Get(event.PoliciesVersion)
+	if err != nil {
+		t.handleError(err)
+		return 0
+	}
+
 	// Short circuit if there are no policies in userland that need filtering.
-	if bitmap&t.config.Policies.FilterableInUserland() == 0 {
-		event.MatchedPoliciesUser = bitmap // store untoched bitmap to be used in sink stage
+	if bitmap&policies.FilterableInUserland() == 0 {
+		event.MatchedPoliciesUser = bitmap // store untouched bitmap to be used in sink stage
 		return bitmap
 	}
 
-	for p := range t.config.Policies.FilterableInUserlandMap() { // range through each userland filterable policy
+	for p := range policies.FilterableInUserlandMap() { // range through each userland filterable policy
 		// Policy ID is the bit offset in the bitmap.
 		bitOffset := uint(p.ID)
 
@@ -449,8 +457,14 @@ func (t *Tracee) processEvents(ctx context.Context, in <-chan *trace.Event) (
 				continue
 			}
 
+			policies, err := policy.Snapshots().Get(event.PoliciesVersion)
+			if err != nil {
+				t.handleError(err)
+				continue
+			}
+
 			// Get a bitmap with all policies containing container filters
-			policiesWithContainerFilter := t.config.Policies.ContainerFilterEnabled()
+			policiesWithContainerFilter := policies.ContainerFilterEnabled()
 
 			// Filter out events that don't have a container ID from all the policies that
 			// have container filters. This will guarantee that any of those policies
@@ -594,8 +608,13 @@ func (t *Tracee) sinkEvents(ctx context.Context, in <-chan *trace.Event) <-chan
 				continue
 			}
 
+			policies, err := policy.Snapshots().Get(event.PoliciesVersion)
+			if err != nil {
+				t.handleError(err)
+				continue
+			}
 			// Populate the event with the names of the matched policies.
-			event.MatchedPolicies = t.config.Policies.MatchedNames(event.MatchedPoliciesUser)
+			event.MatchedPolicies = policies.MatchedNames(event.MatchedPoliciesUser)
 
 			// Parse args here if the rule engine is not enabled (parsed there if it is).
 			if !t.config.EngineConfig.Enabled {

pkg/ebpf/events_pipeline_bench_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/aquasecurity/tracee/pkg/bufferdecoder"
 	"github.com/aquasecurity/tracee/pkg/events"
+	"github.com/aquasecurity/tracee/pkg/utils"
 	"github.com/aquasecurity/tracee/types/trace"
 )
 
@@ -92,6 +93,7 @@ func BenchmarkGetEventFromPool(b *testing.B) {
 				evt.Kubernetes = kubernetesData
 				evt.EventID = int(ctx.EventID)
 				evt.EventName = eventDefinition.GetName()
+				evt.PoliciesVersion = ctx.PoliciesVersion
 				evt.MatchedPoliciesKernel = ctx.MatchedPolicies
 				evt.MatchedPoliciesUser = 0
 				evt.MatchedPolicies = []string{}
@@ -102,6 +104,9 @@ func BenchmarkGetEventFromPool(b *testing.B) {
 				evt.ContextFlags = flags
 				evt.Syscall = syscall
 				evt.Metadata = nil
+				evt.ThreadEntityId = utils.HashTaskID(ctx.HostTid, ctx.StartTime)
+				evt.ProcessEntityId = utils.HashTaskID(ctx.HostPid, ctx.LeaderStartTime)
+				evt.ParentEntityId = utils.HashTaskID(ctx.HostPpid, ctx.ParentStartTime)
 
 				processChan <- evt
 			}
@@ -245,13 +250,17 @@ func BenchmarkNewEventObject(b *testing.B) {
 					Kubernetes:            kubernetesData,
 					EventID:               int(ctx.EventID),
 					EventName:             eventDefinition.GetName(),
+					PoliciesVersion:       ctx.PoliciesVersion,
 					MatchedPoliciesKernel: ctx.MatchedPolicies,
 					ArgsNum:               int(argnum),
 					ReturnValue:           int(ctx.Retval),
 					Args:                  args,
 					StackAddresses:        stackAddresses,
 					ContextFlags:          flags,
 					Syscall:               syscall,
+					ThreadEntityId:        utils.HashTaskID(ctx.HostTid, ctx.StartTime),
+					ProcessEntityId:       utils.HashTaskID(ctx.HostPid, ctx.LeaderStartTime),
+					ParentEntityId:        utils.HashTaskID(ctx.HostPpid, ctx.ParentStartTime),
 				}
 
 				processChan <- &evt

pkg/ebpf/finding.go

@@ -57,6 +57,7 @@ func newEvent(id int, f *detect.Finding, e trace.Event) *trace.Event {
 		ThreadEntityId:        e.ThreadEntityId,
 		ProcessEntityId:       e.ProcessEntityId,
 		ParentEntityId:        e.ParentEntityId,
+		PoliciesVersion:       e.PoliciesVersion,
 		MatchedPoliciesKernel: e.MatchedPoliciesKernel,
 		MatchedPoliciesUser:   e.MatchedPoliciesUser,
 		ArgsNum:               len(arguments),