[CORE] Performance improvements (#132)

Makefile

@@ -79,10 +79,10 @@ ifeq (${CI},true)
 	RACE_FLAG_SYSTEM_TEST := ""
 endif
 
-# DOCKER_HOSTNAME := $(shell hostname)
-# ifneq (${CI},true)
-# 	DOCKER_CMD := DOCKER_HOSTNAME=$(DOCKER_HOSTNAME) $(DOCKER_CMD)
-# endif
+DOCKER_HOSTNAME := $(shell hostname)
+ifneq (${CI},true)
+	DOCKER_CMD := DOCKER_HOSTNAME=$(DOCKER_HOSTNAME) $(DOCKER_CMD)
+endif
 
 all: build
 

README.md

@@ -272,6 +272,6 @@ KubeHound was created by the Adversary Simulation Engineering (ASE) team at Data
 
 With additional support from:
 
-+ Christophe Tafani-Dereeper[@christophetd](https://twitter.com/christophetd)
++ Christophe Tafani-Dereeper [@christophetd](https://twitter.com/christophetd)
 
 We would also like to acknowledge the [BloodHound](https://github.com/BloodHoundAD/BloodHound) team for pioneering the use of graph theory in offensive security and inspiring us to create this project. 

configs/etc/kubehound-dd.yaml

@@ -11,7 +11,7 @@ collector:
   # Live collector configuration
   live:
     # Rate limit of requests/second to the Kubernetes API
-    rate_limit_per_second: 75
+    rate_limit_per_second: 150
 
     # Number of pages to buffer
     page_buffer_size: 50
@@ -43,7 +43,7 @@ janusgraph:
 # Datadog telemetry configuration
 telemetry:
   # Whether to enable Datadog telemetry (default false)
-  enable: true
+  enabled: true
 
   # Default tags to add to all telemetry (free form key-value map)
   tags:
@@ -68,10 +68,10 @@ builder:
 
   # Edge builder configuration
   edge:
-    worker_pool_size: 2
+    worker_pool_size: 5
 
     # Batch size for edge inserts
-    batch_size: 500
+    batch_size: 1000
 
     # Cluster impact batch size for edge inserts
     batch_size_cluster_impact: 10

configs/etc/kubehound-reference.yaml

@@ -2,14 +2,18 @@
 # Reference KubeHound configuration
 #
 
+#
 # K8s collector configuration
+#
 collector:
   # Type of collector to use
   type: live-k8s-api-collector
 
   # Live collector configuration
   live:
     # Rate limit of requests/second to the Kubernetes API
+    # NOTE: most (>90%) of the current runtime of KubeHound is spent in the transfer of data from the remote K8s API server,
+    # and the bulk of that is spent waiting on rate limit. As such increasing this will improve performance roughly linearly.
     rate_limit_per_second: 60
 
     # # Number of entries retrieved by each call on the API (same for all Kubernetes entry types)
@@ -26,7 +30,9 @@ collector:
   #   # Directory holding the K8s json data files
   #   directory: /path/to/directory
 
+#
 # General storage configuration
+#
 storage:
   # Number of connection retries before declaring an error
   retry: 5
@@ -50,10 +56,12 @@ janusgraph:
   # Timeout on requests to the JanusGraph DB instance
   connection_timeout: 30s
 
+#
 # Datadog telemetry configuration
+#
 telemetry:
   # Whether to enable Datadog telemetry (default false)
-  enable: true
+  enabled: true
 
   # Default tags to add to all telemetry (free form key-value map)
   # tags:
@@ -69,7 +77,14 @@ telemetry:
     # URL to send tracer data to the Datadog agent
     url: "127.0.0.1:8226"
 
+#
 # Graph builder configuration
+#
+# NOTE: increasing batch sizes can have some performance improvements by reducing network latency in transferring data
+# between KubeGraph and the application. However, increasing it past a certain level can overload the backend leading 
+# to instability and eventually exceed the size limits of the websocket buffer used to transfer the data. Changing this
+# is not recommended.
+#
 builder:
   # Vertex builder configuration
   # vertex:
@@ -85,9 +100,11 @@ builder:
     large_cluster_optimizations: true
 
     # # Size of the worker pool handling parallel edge inserts
-    # worker_pool_size: 10
+    # # NOTE: this should only be changed if granting additional resources to the KubeGraph container
+    # worker_pool_size: 5
 
     # # Capacity of the worker pool handling parallel edge inserts
+    # # NOTE: this should only be changed in conjunction with the worker_pool_size
     # worker_pool_capacity: 100
 
     # # Batch size for edge inserts

deployments/kubehound/datadog/openmetrics.yaml

@@ -2,7 +2,7 @@ init_config:
     service: kubehound
 
 instances:
-  - prometheus_url: http://janusgraph:8099/metrics
+  - prometheus_url: http://kubegraph:8099/metrics
     namespace: "kubehound.janusgraph"
     type_overrides: 
       metrics_org_apache_tinkerpop_gremlin_server_GremlinServer_gremlin_groovy_sessionless_class_cache_miss_rate_Number: "gauge"
@@ -19,7 +19,6 @@ instances:
       java_lang_ClassLoading_TotalLoadedClassCount: "gauge"
       java_lang_CodeHeap_non_nmethods_Usage_used: "gauge"
       java_lang_G1_Survivor_Space_CollectionUsage_init: "gauge"
-      metrics_org_apache_tinkerpop_gremlin_server_GremlinServer_gremlin_groovy_sessionless_class_cache_miss_rate_Number: "gauge"
       java_lang_G1_Survivor_Space_CollectionUsageThreshold: "gauge"
       metrics_org_apache_tinkerpop_gremlin_server_GremlinServer_gremlin_groovy_sessionless_class_cache_load_failure_rate_Number: "gauge"
       java_lang_Compressed_Class_Space_UsageThresholdSupported: "gauge"

deployments/kubehound/docker-compose.datadog.yaml

@@ -6,7 +6,7 @@ services:
     profiles: ["infra"]
     container_name: ${COMPOSE_PROJECT_NAME}-datadog-agent
     ports:
-      - "127.0.0.1:8225:8125"
+      - "127.0.0.1:8225:8125/UDP"
       - "127.0.0.1:8226:8126"
     environment:
       - DD_API_KEY=${DD_API_KEY:?error}
@@ -17,6 +17,7 @@ services:
       - DD_LOGS_ENABLED=true
       - DD_PROCESS_AGENT_ENABLED=true
       - DD_APM_NON_LOCAL_TRAFFIC=true
+      - DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true
       - DD_CONTAINER_LABELS_AS_TAGS={"com.docker.compose.service":"+app"}
       - DD_CONTAINER_INCLUDE=name:kubehound-*
       # https://github.com/DataDog/datadog-agent/issues/6599

deployments/kubehound/kubegraph/Dockerfile

@@ -31,8 +31,13 @@ COPY --chown=janusgraph:janusgraph kubehound-dsl-init.groovy ${JANUS_HOME}/scrip
 # Set JVM configuration
 ENV JAVA_OPTIONS_FILE ${JANUS_HOME}/conf/jvm.options
 
+# Use an in-memory backend for speed
+ENV JANUS_PROPS_TEMPLATE=inmemory
+
 # Optimize for writes
-ENV janusgraph.ids.block-size=3000000
+ENV janusgraph.ids.block-size=1000000000
+ENV janusgraph.ids.renew-percentage 0.3
+ENV storage.batch-loading=true
 
 # Enforce strict schema constraints as per https://docs.janusgraph.org/configs/configuration-reference/#schema
 ENV janusgraph.schema.constraints=true
@@ -55,7 +60,7 @@ ENV gremlinserver.metrics.csvReporter.enabled=false
 # gremlinPool will default to Runtime.availableProcessors()
 ENV gremlinserver.gremlinPool=0 
 # threadPoolWorker should be 2x VCPU (TODO: can we set dynamically?)
-ENV gremlinserver.threadPoolWorker=8
+ENV gremlinserver.threadPoolWorker=16
 
 # Custom SCRIPT plugin for DSL support
 ENV gremlinserver.scriptEngines.gremlin-groovy.plugins[org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin].classImports[+]=com.datadog.ase.kubehound.KubeHoundTraversalSource

docs/faq.md

@@ -6,9 +6,9 @@ As always.. it depends :) The bulk of the work comes in building up the attack g
 
 | Cluster Size (Pods) | Duration |
 | --------------------|----------|
-| 1,000 | 2 mins |
-| 10,000 | 15 mins |
-| 30,000 | 25 mins |
+| 1,000 | <1 min |
+| 10,000 | 3 mins |
+| 30,000 | 7 mins |
 
 **What happens when you run KubeHound multiple times?**
 

go.mod

@@ -3,126 +3,130 @@ module github.com/DataDog/KubeHound
 go 1.20
 
 require (
-	github.com/DataDog/datadog-go/v5 v5.1.1
+	github.com/DataDog/datadog-go/v5 v5.3.0
 	github.com/alitto/pond v1.8.3
-	github.com/apache/tinkerpop/gremlin-go/v3 v3.6.4
+	github.com/apache/tinkerpop/gremlin-go/v3 v3.7.0
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/spf13/cobra v1.6.1
-	github.com/stretchr/testify v1.8.2
-	go.mongodb.org/mongo-driver v1.11.6
+	github.com/stretchr/testify v1.8.4
+	go.mongodb.org/mongo-driver v1.12.1
 	go.uber.org/ratelimit v0.2.0
-	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
-	gopkg.in/DataDog/dd-trace-go.v1 v1.51.0
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	gopkg.in/DataDog/dd-trace-go.v1 v1.55.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.27.2
-	k8s.io/apimachinery v0.27.2
+	k8s.io/api v0.28.2
+	k8s.io/apimachinery v0.28.2
 	sigs.k8s.io/controller-runtime v0.15.0
 )
 
 require (
 	github.com/DataDog/appsec-internal-go v1.0.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/obfuscate v0.45.0-rc.1 // indirect
-	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.45.0-rc.1 // indirect
-	github.com/DataDog/go-libddwaf v1.2.0 // indirect
-	github.com/DataDog/go-tuf v0.3.0--fix-localmeta-fork // indirect
-	github.com/DataDog/gostackparse v0.5.0 // indirect
-	github.com/DataDog/sketches-go v1.3.0 // indirect
-	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/DataDog/datadog-agent/pkg/obfuscate v0.46.0 // indirect
+	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.0-devel.0.20230725154044-2549ba9058df // indirect
+	github.com/DataDog/go-libddwaf v1.5.0 // indirect
+	github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect
+	github.com/DataDog/gostackparse v0.7.0 // indirect
+	github.com/DataDog/sketches-go v1.4.2 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/ebitengine/purego v0.5.0-alpha.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
-	github.com/go-openapi/jsonreference v0.20.1 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/hashicorp/hcl v1.0.1-vault-5 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.15.0 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/montanaflynn/stats v0.6.6 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nicksnyder/go-i18n/v2 v2.2.1 // indirect
-	github.com/outcaste-io/ristretto v0.2.1 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
-	github.com/philhofer/fwd v1.1.1 // indirect
+	github.com/outcaste-io/ristretto v0.2.3 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.15.1 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 // indirect
-	github.com/secure-systems-lab/go-securesystemslib v0.5.0 // indirect
+	github.com/sagikazarmark/locafero v0.3.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/afero v1.9.3 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/afero v1.10.0 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/objx v0.5.0 // indirect
-	github.com/subosito/gotenv v1.4.2 // indirect
-	github.com/tinylib/msgp v1.1.6 // indirect
+	github.com/stretchr/objx v0.5.1 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tinylib/msgp v1.1.8 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.1.1 // indirect
-	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
-	go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	go4.org/intern v0.0.0-20230525184215-6c62f75575cb // indirect
+	go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
-	golang.org/x/oauth2 v0.5.0 // indirect
-	golang.org/x/sync v0.2.0 // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/oauth2 v0.12.0 // indirect
+	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
-	google.golang.org/grpc v1.53.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	inet.af/netaddr v0.0.0-20220811202034-502d2d690317 // indirect
+	inet.af/netaddr v0.0.0-20230525184311-b8eac61e914a // indirect
 	k8s.io/apiextensions-apiserver v0.27.2 // indirect
 	k8s.io/component-base v0.27.2 // indirect
-	k8s.io/klog/v2 v2.90.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
-	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 require (
-	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/viper v1.15.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/viper v1.17.0
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
-	k8s.io/client-go v0.27.2
+	k8s.io/client-go v0.28.2
 )