Change summary file aggregation logic summary #171 (#545)

Signed-off-by: Eswar Rajan Subramanian <eswar@accuknox.com>

src/systempolicy/pathAggregator.go → src/common/pathAggregator.go

@@ -1,4 +1,4 @@
-package systempolicy
+package common
 
 import (
 	"path/filepath"
@@ -53,6 +53,12 @@ type HTTPDst struct {
 	HTTPTree    map[string]map[string]*Node
 }
 
+// SysPath Structure
+type SysPath struct {
+	Path  string
+	IsDir bool
+}
+
 func (n *Node) generatePaths(results map[string]bool, parentPath string) {
 	for _, childNode := range n.childNodes {
 		childNode.generatePaths(results, parentPath+n.path)
@@ -248,7 +254,7 @@ func AggregatePaths(paths []string) []SysPath {
 		}
 		sysPath := SysPath{
 			Path:  path,
-			isDir: isDir,
+			IsDir: isDir,
 		}
 		results = append(results, sysPath)
 	}
@@ -296,7 +302,7 @@ func MergeAndAggregatePaths(dirs []string, paths []string) []SysPath {
 	for path, isDir := range aggregatedPaths {
 		sysPath := SysPath{
 			Path:  path,
-			isDir: isDir,
+			IsDir: isDir,
 		}
 		results = append(results, sysPath)
 	}
@@ -312,7 +318,7 @@ func AggregatePathsExt(paths []string) []string {
 	var flist []string
 	for _, sp := range results {
 		rec := sp.Path
-		if sp.isDir {
+		if sp.IsDir {
 			if !strings.HasSuffix(rec, "/") {
 				rec = rec + "/"
 			}

src/systempolicy/pathAggregator_test.go → src/common/pathAggregator_test.go

@@ -1,4 +1,4 @@
-package systempolicy
+package common
 
 import (
 	"testing"
@@ -16,8 +16,8 @@ func TestAggregatePaths_1(t *testing.T) {
 	results := AggregatePaths(paths)
 
 	assert.Equal(t, len(results), 2)
-	assert.False(t, results[0].isDir)
-	assert.False(t, results[1].isDir)
+	assert.False(t, results[0].IsDir)
+	assert.False(t, results[1].IsDir)
 }
 
 func TestAggregatePaths_2(t *testing.T) {
@@ -31,7 +31,7 @@ func TestAggregatePaths_2(t *testing.T) {
 	results := AggregatePaths(paths)
 
 	assert.Equal(t, len(results), 1)
-	assert.True(t, results[0].isDir)
+	assert.True(t, results[0].IsDir)
 }
 
 func TestAggregatePaths_3(t *testing.T) {

src/observability/kubearmor.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/accuknox/auto-policy-discovery/src/cluster"
+	"github.com/accuknox/auto-policy-discovery/src/common"
 	"github.com/accuknox/auto-policy-discovery/src/libs"
 	opb "github.com/accuknox/auto-policy-discovery/src/protobuf/v1/observability"
 	"github.com/accuknox/auto-policy-discovery/src/types"
@@ -235,12 +236,35 @@ func aggregateProcFileData(data []types.SysObsProcFileData) []types.SysObsProcFi
 	if len(data) <= 0 {
 		return nil
 	}
+
+	var destPaths, aggregatedDir []string
+	for _, locData := range data {
+		destPaths = append(destPaths, locData.Destination)
+	}
+	aggregatedSysPath := common.AggregatePaths(destPaths)
+
+	for _, sp := range aggregatedSysPath {
+		if sp.IsDir {
+			aggregatedDir = append(aggregatedDir, sp.Path)
+		}
+	}
+
 	res := []types.SysObsProcFileData{}
 
 	for _, locData := range data {
+		var destination string
+
+		for _, dir := range aggregatedDir {
+			if strings.HasPrefix(locData.Destination, dir) {
+				destination = dir
+				break
+			}
+			destination = locData.Destination
+		}
+
 		locKey := types.SysObsProcFileMapKey{
 			Source:      locData.Source,
-			Destination: locData.Destination[:strings.LastIndex(locData.Destination, "/")+1],
+			Destination: destination,
 			Status:      locData.Status,
 		}
 
@@ -324,7 +348,6 @@ func GetKubearmorSummaryData(req *opb.Request) ([]types.SysObsProcFileData, []ty
 	}
 
 	if req.Aggregate {
-		processData = aggregateProcFileData(processData)
 		fileData = aggregateProcFileData(fileData)
 	}
 

src/systempolicy/deduplicator.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/accuknox/auto-policy-discovery/src/common"
 	"github.com/accuknox/auto-policy-discovery/src/config"
 	"github.com/accuknox/auto-policy-discovery/src/libs"
 	types "github.com/accuknox/auto-policy-discovery/src/types"
@@ -111,12 +112,12 @@ func UpdateProcessOperation(newPolicy types.KnoxSystemPolicy, existingPolicies [
 		}
 	}
 
-	mergedSysPaths := MergeAndAggregatePaths(dirs, paths)
+	mergedSysPaths := common.MergeAndAggregatePaths(dirs, paths)
 
 	// step 4: init and updated proecss spec
 	newPolicy.Spec.Process = types.KnoxSys{} // init
 	for _, pathSpec := range mergedSysPaths {
-		if pathSpec.isDir {
+		if pathSpec.IsDir {
 			matchDirs := types.KnoxMatchDirectories{
 				Dir: pathSpec.Path,
 			}
@@ -251,12 +252,12 @@ func UpdateFileOperation(newPolicy types.KnoxSystemPolicy, existingPolicies []ty
 		}
 	}
 
-	mergedSysPaths := MergeAndAggregatePaths(dirs, paths)
+	mergedSysPaths := common.MergeAndAggregatePaths(dirs, paths)
 
 	// step 4: init and updated file spec
 	newPolicy.Spec.File = types.KnoxSys{} // init
 	for _, pathSpec := range mergedSysPaths {
-		if pathSpec.isDir {
+		if pathSpec.IsDir {
 			matchDirs := types.KnoxMatchDirectories{
 				Dir: pathSpec.Path,
 			}

src/systempolicy/systemPolicy.go

@@ -16,6 +16,7 @@ import (
 	"sigs.k8s.io/yaml"
 
 	"github.com/accuknox/auto-policy-discovery/src/cluster"
+	"github.com/accuknox/auto-policy-discovery/src/common"
 	cfg "github.com/accuknox/auto-policy-discovery/src/config"
 	fc "github.com/accuknox/auto-policy-discovery/src/feedconsumer"
 	"github.com/accuknox/auto-policy-discovery/src/libs"
@@ -162,12 +163,6 @@ type SysLogKey struct {
 	PodName   string
 }
 
-// SysPath Structure
-type SysPath struct {
-	Path  string
-	isDir bool
-}
-
 // ================ //st
 // == System Log == //
 // ================ //
@@ -480,7 +475,7 @@ func discoverFileOperationPolicy(results []types.KnoxSystemPolicy, pod types.Pod
 
 	// step 3: aggregate file paths
 	for src, filePaths := range srcToDest {
-		aggregatedFilePaths := AggregatePaths(filePaths)
+		aggregatedFilePaths := common.AggregatePaths(filePaths)
 
 		// step 4: append spec to the policy
 		for _, filePath := range aggregatedFilePaths {
@@ -524,7 +519,7 @@ func discoverProcessOperationPolicy(results []types.KnoxSystemPolicy, pod types.
 
 	// step 3: aggregate process paths
 	for src, processPaths := range srcToDest {
-		aggregatedProcessPaths := AggregatePaths(processPaths)
+		aggregatedProcessPaths := common.AggregatePaths(processPaths)
 
 		// step 4: append spec to the policy
 		for _, processPath := range aggregatedProcessPaths {
@@ -808,9 +803,9 @@ func ConvertWPFSToKnoxSysPolicy(wpfsSet types.ResourceSetMap, pnMap types.Policy
 		policy.Metadata["type"] = wpfs.SetType
 
 		for _, fpath := range fsset {
-			path := SysPath{
+			path := common.SysPath{
 				Path:  fpath,
-				isDir: strings.HasSuffix(fpath, "/"),
+				IsDir: strings.HasSuffix(fpath, "/"),
 			}
 			src := ""
 			if wpfs.SetType == SYS_OP_NETWORK || strings.HasPrefix(wpfs.FromSource, "/") {
@@ -870,7 +865,7 @@ func buildSystemPolicy() types.KnoxSystemPolicy {
 	}
 }
 
-func updateSysPolicySpec(opType string, policy types.KnoxSystemPolicy, src string, pathSpec SysPath) types.KnoxSystemPolicy {
+func updateSysPolicySpec(opType string, policy types.KnoxSystemPolicy, src string, pathSpec common.SysPath) types.KnoxSystemPolicy {
 	if opType == SYS_OP_NETWORK {
 		matchProtocols := types.KnoxMatchProtocols{
 			Protocol: pathSpec.Path,
@@ -885,7 +880,7 @@ func updateSysPolicySpec(opType string, policy types.KnoxSystemPolicy, src strin
 		return policy
 	}
 	// matchDirectories
-	if pathSpec.isDir {
+	if pathSpec.IsDir {
 		path := pathSpec.Path
 		if !strings.HasSuffix(path, "/") {
 			path = path + "/"
@@ -1298,7 +1293,7 @@ func GenFileSetForAllPodsInCluster(clusterName string, pods []types.Pod, settype
 		mergedfs = removeDuplicates(append(fs, out[wpfs]...))
 		if !isNetworkOp {
 			// Path aggregation makes sense for file, process operations only
-			mergedfs = AggregatePathsExt(mergedfs) // merge and sort the filesets
+			mergedfs = common.AggregatePathsExt(mergedfs) // merge and sort the filesets
 		}
 
 		// Add/Update DB Entry