perf: introduce stack cache

We hash stacks so that we can cache them when emitting data in the
binary format.

src/cache.c

@@ -48,7 +48,8 @@ queue_item__destroy(queue_item_t * self, void (*deallocator)(value_t)) {
  if (!isvalid(self))
  return;
 
- deallocator(self->value);
+ if (deallocator)
+ deallocator(self->value);
 
  free(self);
 }

src/events.h

@@ -75,10 +75,10 @@
  } \
  }
 
-#define emit_stack(format, pid, iid, tid, ...)  \
+#define emit_stack(hash, format, pid, iid, tid, ...) \
  { \
  if (pargs.binary) { \
- mojo_stack(pid, iid, tid);  \
+ mojo_stack(hash, pid, iid, tid); \
  } else { \
  fprintfp(pargs.output_file, format, pid, iid, tid, __VA_ARGS__); \
  } \

src/mojo.h

@@ -29,7 +29,7 @@
 #include "cache.h"
 #include "platform.h"
 
-#define MOJO_VERSION 3
+#define MOJO_VERSION 4
 
 enum {
  MOJO_RESERVED,
@@ -44,7 +44,7 @@ enum {
  MOJO_METRIC_TIME,
  MOJO_METRIC_MEMORY,
  MOJO_STRING,
- MOJO_STRING_REF,
+ MOJO_STACK_REF,
  MOJO_MAX,
 };
 
@@ -119,10 +119,11 @@ static inline void mojo_integer(mojo_int_t integer, int sign) {
  mojo_string(label); \
  mojo_fstring(__VA_ARGS__);
 
-#define mojo_stack(pid, iid, tid) \
- mojo_event(MOJO_STACK); \
- mojo_integer(pid, 0); \
- mojo_integer(iid, 0); \
+#define mojo_stack(key, pid, iid, tid) \
+ mojo_event(MOJO_STACK); \
+ mojo_ref(key); \
+ mojo_integer(pid, 0); \
+ mojo_integer(iid, 0); \
  mojo_fstring(FORMAT_TID, tid);
 
 #define mojo_frame(frame) \
@@ -160,4 +161,11 @@ static inline void mojo_integer(mojo_int_t integer, int sign) {
  mojo_event(MOJO_STRING_REF); \
  mojo_ref(key);
 
+#define mojo_stack_ref(key, pid, iid, tid) \
+ mojo_event(MOJO_STACK_REF); \
+ mojo_ref(key); \
+ mojo_integer(pid, 0); \
+ mojo_integer(iid, 0); \
+ mojo_fstring(FORMAT_TID, tid);
+
 #endif

src/py_proc.c

@@ -54,6 +54,9 @@
 #include "py_thread.h"
 
 
+#define MAX_STACK_CACHE_SIZE (1 << 16) // 64K
+
+
 // ---- PRIVATE ---------------------------------------------------------------
 
 #define py_proc__memcpy(self, raddr, size, dest) copy_memory(self->proc_ref, raddr, size, dest)
@@ -708,6 +711,15 @@ py_proc_new(int child) {
 
  py_proc->frames_heap = py_proc->frames = NULL_MEM_BLOCK;
 
+ py_proc->stack_cache = lru_cache_new(MAX_STACK_CACHE_SIZE, NULL);
+ if (!isvalid(py_proc->stack_cache)) {
+ log_e("Failed to allocate stack cache");
+ goto error;
+ }
+ #ifdef DEBUG
+ py_proc->stack_cache->name = "stack cache";
+ #endif
+
  py_proc->frame_cache = lru_cache_new(MAX_FRAME_CACHE_SIZE, (void (*)(value_t)) frame__destroy);
  if (!isvalid(py_proc->frame_cache)) {
  log_e("Failed to allocate frame cache");
@@ -1175,7 +1187,7 @@ _py_proc__sample_interpreter(py_proc_t * self, PyInterpreterState * is, ctime_t
  }
  }
 
- py_thread__emit_collapsed_stack(
+ py_thread__emit_sample(
  &py_thread,
  interp_id,
  time_delta,
@@ -1327,6 +1339,7 @@ py_proc__destroy(py_proc_t * self) {
 
  lru_cache__destroy(self->string_cache);
  lru_cache__destroy(self->frame_cache);
+ lru_cache__destroy(self->stack_cache);
 
  free(self);
 }

src/py_proc.h

@@ -75,6 +75,7 @@ typedef struct {
 
  void * is_raddr;
 
+ lru_cache_t * stack_cache;
  lru_cache_t * frame_cache;
  lru_cache_t * string_cache;
 

src/py_thread.c

@@ -444,8 +444,6 @@ _py_thread__unwind_iframe_stack(py_thread_t * self, void * iframe_raddr) {
  break;
  }
  }
-
- invalid = fail(_py_thread__resolve_py_stack(self)) || invalid;
 
  return invalid;
 }
@@ -893,7 +891,7 @@ py_thread__next(py_thread_t * self) {
 
 // ----------------------------------------------------------------------------
 void
-py_thread__emit_collapsed_stack(py_thread_t * self, int64_t interp_id, ctime_t time_delta, ssize_t mem_delta) {
+py_thread__emit_sample(py_thread_t * self, int64_t interp_id, ctime_t time_delta, ssize_t mem_delta) {
  if (!pargs.full && pargs.memory && mem_delta == 0)
  return;
 
@@ -922,18 +920,8 @@ py_thread__emit_collapsed_stack(py_thread_t * self, int64_t interp_id, ctime_t t
  }
  }
 
- // Group entries by thread.
- emit_stack(
- pargs.head_format, self->proc->pid, interp_id, self->tid,
- // These are relevant only in `where` mode
- is_idle ? "💤" : "🚀",
- self->proc->child ? "🧒" : ""
- );
-
  int error = FALSE;
-
  #ifdef NATIVE
-
  // We sample the kernel frame stack BEFORE interrupting because otherwise
  // we would see the ptrace syscall call stack, which is not very interesting.
  // The downside is that the kernel stack might not be in sync with the other
@@ -953,6 +941,7 @@ py_thread__emit_collapsed_stack(py_thread_t * self, int64_t interp_id, ctime_t t
 
  V_DESC(self->proc->py_v);
 
+ stack_hash_t stack_hash = 0;
  if (isvalid(self->top_frame)) {
  if (V_MIN(3, 11)) {
  if (fail(_py_thread__unwind_cframe_stack(self))) {
@@ -966,11 +955,41 @@ py_thread__emit_collapsed_stack(py_thread_t * self, int64_t interp_id, ctime_t t
  error = TRUE;
  }
  }
-
- if (fail(_py_thread__resolve_py_stack(self))) {
- emit_invalid_frame();
- error = TRUE;
+
+ stack_hash = stack_py_hash();
+ #ifdef NATIVE
+ stack_hash ^= stack_native_hash();
+ if (pargs.kernel) {
+ stack_hash ^= stack_kernel_hash();
  }
+ #endif
+
+ if (pargs.binary) {
+ value_t seen_stack = lru_cache__maybe_hit(self->proc->stack_cache, stack_hash);
+ if (seen_stack) {
+ mojo_stack_ref(stack_hash, self->proc->pid, interp_id, self->tid);
+ goto finish_sample;
+ } else {
+ lru_cache__store(self->proc->stack_cache, stack_hash, (value_t)TRUE);
+ }
+ }
+ }
+
+ // Group entries by thread.
+ emit_stack(
+ stack_hash,
+ pargs.head_format, self->proc->pid, interp_id, self->tid,
+ // These are relevant only in `where` mode
+ is_idle ? "💤" : "🚀",
+ self->proc->child ? "🧒" : ""
+ );
+
+ if (stack_hash == 0)
+ // We have no stack to emit.
+ goto finish_sample;
+
+ if (!error && fail(_py_thread__resolve_py_stack(self))) {
+ emit_invalid_frame();
  }
 
  #ifdef NATIVE
@@ -1036,6 +1055,7 @@ py_thread__emit_collapsed_stack(py_thread_t * self, int64_t interp_id, ctime_t t
  }
  #endif
 
+finish_sample:
  if (pargs.gc && py_proc__is_gc_collecting(self->proc) == TRUE) {
  emit_gc();
  stats_gc_time(time_delta);
@@ -1060,7 +1080,7 @@ py_thread__emit_collapsed_stack(py_thread_t * self, int64_t interp_id, ctime_t t
  stats_count_sample();
  if (error) stats_count_error();
  stats_check_duration(stopwatch_duration());
-} /* py_thread__emit_collapsed_stack */
+} /* py_thread__emit_sample */
 
 
 // ----------------------------------------------------------------------------

src/py_thread.h

@@ -97,7 +97,7 @@ py_thread__next(py_thread_t *);
  * @param ssize_t the memory delta.
  */
 void
-py_thread__emit_collapsed_stack(py_thread_t *, int64_t, ctime_t, ssize_t);
+py_thread__emit_sample(py_thread_t *, int64_t, ctime_t, ssize_t);
 
 
 /**

src/stack.h

@@ -51,8 +51,14 @@ typedef struct {
  #endif
 } stack_dt;
 
+typedef key_dt stack_hash_t;
+
 static stack_dt * _stack;
 
+#define ROTL(x) ((x << 1) | (x >> (sizeof(x) * CHAR_BIT - 1)))
+
+
+// ----------------------------------------------------------------------------
 static inline int
 stack_allocate(size_t size) {
  if (isvalid(_stack))
@@ -73,6 +79,8 @@ stack_allocate(size_t size) {
  SUCCESS;
 }
 
+
+// ----------------------------------------------------------------------------
 static inline void
 stack_deallocate(void) {
  if (!isvalid(_stack))
@@ -89,7 +97,7 @@ stack_deallocate(void) {
 }
 
 
-
+// ----------------------------------------------------------------------------
 static inline int
 stack_has_cycle(void) {
  if (_stack->pointer < 2)
@@ -110,6 +118,8 @@ stack_has_cycle(void) {
  return FALSE;
 }
 
+
+// ----------------------------------------------------------------------------
 static inline void
 stack_py_push(void * origin, void * code, int lasti) {
  _stack->py_base[_stack->pointer++] = (py_frame_t) {
@@ -119,6 +129,55 @@ stack_py_push(void * origin, void * code, int lasti) {
  };
 }
 
+
+// ----------------------------------------------------------------------------
+static inline stack_hash_t
+stack_py_hash(void) {
+ stack_hash_t hash = 0;
+
+ for (ssize_t i = 0; i < _stack->pointer; i++) {
+ py_frame_t * frame = _stack->py_base+i;
+ hash = ROTL(hash) ^ py_frame_key(frame->code, frame->lasti);
+ }
+
+ return hash;
+}
+
+
+#ifdef NATIVE
+// ----------------------------------------------------------------------------
+static inline stack_hash_t
+stack_native_hash(void) {
+ stack_hash_t hash = 0;
+
+ for (ssize_t i = 0; i < _stack->native_pointer; i++) {
+ frame_t * frame = _stack->native_base[i];
+ hash = ROTL(hash) ^ frame->key;
+ }
+
+ return hash;
+}
+
+
+// ----------------------------------------------------------------------------
+static inline stack_hash_t
+stack_kernel_hash(void) {
+ stack_hash_t hash = 0;
+
+ for (ssize_t i = 0; i < _stack->kernel_pointer; i++) {
+ key_dt frame = (key_dt)_stack->kernel_base[i];
+ hash = ROTL(hash) ^ frame;
+ }
+
+ return hash;
+}
+
+#endif
+
+
+// ----------------------------------------------------------------------------
+
+
 #define stack_pointer() (_stack->pointer)
 #define stack_push(frame) {_stack->base[_stack->pointer++] = frame;}
 #define stack_set(i, frame) {_stack->base[i] = frame;}