augment QEMU to support whole-system fuzzing in AFL.

qemu_mode/qemu/afl-qemu-cpu-inl.h

@@ -27,6 +27,7 @@
  */
 
 #include <sys/shm.h>
+#include "afl.h"
 #include "../../config.h"
 
 /***************************
@@ -46,12 +47,13 @@
    _start and does the usual forkserver stuff, not very different from
    regular instrumentation injected via afl-as.h. */
 
-#define AFL_QEMU_CPU_SNIPPET2 do { \
-    if(tb->pc == afl_entry_point) { \
+#define AFL_QEMU_CPU_SNIPPET2(env, pc) do { \
+    if(pc == afl_entry_point && pc && getenv("AFLGETWORK") == 0) { \
       afl_setup(); \
       afl_forkserver(env); \
+      aflStart = 1; \
     } \
-    afl_maybe_log(tb->pc); \
+    afl_maybe_log(pc); \
   } while (0)
 
 /* We use one additional file descriptor to relay "needs translation"
@@ -61,17 +63,27 @@
 
 /* This is equivalent to afl-as.h: */
 
-static unsigned char *afl_area_ptr;
+static unsigned char *afl_area_ptr = 0;
 
 /* Exported variables populated by the code patched into elfload.c: */
 
-abi_ulong afl_entry_point, /* ELF entry point (_start) */
-          afl_start_code,  /* .text start pointer      */
-          afl_end_code;    /* .text end pointer        */
+target_ulong afl_entry_point = 0, /* ELF entry point (_start) */
+          afl_start_code = 0,  /* .text start pointer      */
+          afl_end_code = 0;    /* .text end pointer        */
+
+int aflStart = 0;               /* we've started fuzzing */
+int aflEnableTicks = 0;         /* re-enable ticks for each test */
+int aflGotLog = 0;              /* we've seen dmesg logging */
+
+/* from command line options */
+const char *aflFile = "/tmp/work";
+unsigned long aflPanicAddr = (unsigned long)-1;
+unsigned long aflDmesgAddr = (unsigned long)-1;
 
 /* Set in the child process in forkserver mode: */
 
-static unsigned char afl_fork_child;
+unsigned char afl_fork_child = 0;
+int afl_wants_cpu_to_stop = 0;
 unsigned int afl_forksrv_pid;
 
 /* Instrumentation ratio: */
@@ -80,9 +92,7 @@ static unsigned int afl_inst_rms = MAP_SIZE;
 
 /* Function declarations. */
 
-static void afl_setup(void);
-static void afl_forkserver(CPUArchState*);
-static inline void afl_maybe_log(abi_ulong);
+static inline void afl_maybe_log(target_ulong);
 
 static void afl_wait_tsl(CPUArchState*, int);
 static void afl_request_tsl(target_ulong, target_ulong, uint64_t);
@@ -104,10 +114,9 @@ struct afl_tsl {
  * ACTUAL IMPLEMENTATION *
  *************************/
 
-
 /* Set up SHM region and initialize other stuff. */
 
-static void afl_setup(void) {
+void afl_setup(void) {
 
   char *id_str = getenv(SHM_ENV_VAR),
        *inst_r = getenv("AFL_INST_RATIO");
@@ -145,16 +154,23 @@ static void afl_setup(void) {
   if (getenv("AFL_INST_LIBS")) {
 
     afl_start_code = 0;
-    afl_end_code   = (abi_ulong)-1;
+    afl_end_code   = (target_ulong)-1;
 
   }
 
 }
 
+static ssize_t uninterrupted_read(int fd, void *buf, size_t cnt)
+{
+    ssize_t n;
+    while((n = read(fd, buf, cnt)) == -1 && errno == EINTR)
+        continue;
+    return n;
+}
 
 /* Fork server logic, invoked once we hit _start. */
 
-static void afl_forkserver(CPUArchState *env) {
+void afl_forkserver(CPUArchState *env) {
 
   static unsigned char tmp[4];
 
@@ -176,7 +192,7 @@ static void afl_forkserver(CPUArchState *env) {
 
     /* Whoops, parent dead? */
 
-    if (read(FORKSRV_FD, tmp, 4) != 4) exit(2);
+    if (uninterrupted_read(FORKSRV_FD, tmp, 4) != 4) exit(2);
 
     /* Establish a channel with child to grab translation commands. We'll 
        read from t_fd[0], child will write to TSL_FD. */
@@ -218,34 +234,58 @@ static void afl_forkserver(CPUArchState *env) {
 
 }
 
-
-/* The equivalent of the tuple logging routine from afl-as.h. */
-
-static inline void afl_maybe_log(abi_ulong cur_loc) {
-
-  static __thread abi_ulong prev_loc;
+static inline target_ulong aflHash(target_ulong cur_loc)
+{
+  if(!aflStart)
+    return 0;
 
   /* Optimize for cur_loc > afl_end_code, which is the most likely case on
      Linux systems. */
 
   if (cur_loc > afl_end_code || cur_loc < afl_start_code || !afl_area_ptr)
-    return;
+    return 0;
+
+#ifdef DEBUG_EDGES
+  if(1) {
+    printf("exec %lx\n", cur_loc);
+    fflush(stdout);
+  }
+#endif
 
   /* Looks like QEMU always maps to fixed locations, so ASAN is not a
      concern. Phew. But instruction addresses may be aligned. Let's mangle
      the value to get something quasi-uniform. */
 
-  cur_loc  = (cur_loc >> 4) ^ (cur_loc << 8);
-  cur_loc &= MAP_SIZE - 1;
+  target_ulong h = cur_loc;
+  h ^= cur_loc >> 33;
+  h *= 0xff51afd7ed558ccd;
+  h ^= h >> 33;
+  h *= 0xc4ceb9fe1a85ec53;
+  h ^= h >> 33;
+
+  h &= MAP_SIZE - 1;
 
   /* Implement probabilistic instrumentation by looking at scrambled block
      address. This keeps the instrumented locations stable across runs. */
 
-  if (cur_loc >= afl_inst_rms) return;
+  if (h >= afl_inst_rms) return 0;
+  return h;
+}
+
+/* todo: generate calls to helper_aflMaybeLog during translation */
+static inline void helper_aflMaybeLog(target_ulong cur_loc) {
+  static __thread target_ulong prev_loc;
 
   afl_area_ptr[cur_loc ^ prev_loc]++;
   prev_loc = cur_loc >> 1;
+}
+
+/* The equivalent of the tuple logging routine from afl-as.h. */
 
+static inline void afl_maybe_log(target_ulong cur_loc) {
+  cur_loc = aflHash(cur_loc);
+  if(cur_loc)
+    helper_aflMaybeLog(cur_loc);
 }
 
 
@@ -284,7 +324,25 @@ static void afl_wait_tsl(CPUArchState *env, int fd) {
     if (read(fd, &t, sizeof(struct afl_tsl)) != sizeof(struct afl_tsl))
       break;
 
-    tb_find_slow(env, t.pc, t.cs_base, t.flags);
+    if(0 && env) {
+#ifdef CONFIG_USER_ONLY
+        tb_find_slow(env, t.pc, t.cs_base, t.flags);
+#else
+        /* if the child system emulator pages in new code and then JITs it, 
+        and sends its address to the server, the server cannot also JIT it 
+        without having it's guest's kernel page the data in !  
+        so we will only JIT kernel code segment which shouldnt page.
+        */
+        if(t.pc >= 0xffffffff81000000 && t.pc <= 0xffffffff81ffffff) {
+            //printf("wait_tsl %lx -- jit\n", t.pc); fflush(stdout);
+            tb_find_slow(env, t.pc, t.cs_base, t.flags);
+        } else {
+            //printf("wait_tsl %lx -- ignore nonkernel\n", t.pc); fflush(stdout);
+        }
+#endif
+    } else {
+        //printf("wait_tsl %lx -- ignore\n", t.pc); fflush(stdout);
+    }
 
   }
 

qemu_mode/qemu/afl.h

@@ -0,0 +1,16 @@
+
+extern const char *aflFile;
+extern unsigned long aflPanicAddr;
+extern unsigned long aflDmesgAddr;
+
+extern int aflEnableTicks;
+extern int aflStart;
+extern int aflGotLog;
+extern target_ulong afl_start_code, afl_end_code;
+extern unsigned char afl_fork_child;
+extern int afl_wants_cpu_to_stop;
+
+void afl_setup(void);
+void afl_forkserver(CPUArchState*);
+
+

qemu_mode/qemu/cpu-exec.c

@@ -198,6 +198,7 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr)
 #endif /* DEBUG_DISAS */
 
     cpu->can_do_io = 0;
+    target_ulong pc = env->eip;
     next_tb = tcg_qemu_tb_exec(env, tb_ptr);
     cpu->can_do_io = 1;
     trace_exec_tb_exit((void *) (next_tb & ~TB_EXIT_MASK),
@@ -216,13 +217,19 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr)
             assert(cc->set_pc);
             cc->set_pc(cpu, tb->pc);
         }
+    } else {
+        /* we executed it, trace it */
+        AFL_QEMU_CPU_SNIPPET2(env, pc);
     }
+
     if ((next_tb & TB_EXIT_MASK) == TB_EXIT_REQUESTED) {
         /* We were asked to stop executing TBs (probably a pending
          * interrupt. We've now stopped, so clear the flag.
          */
         cpu->tcg_exit_req = 0;
     }
+    if(afl_wants_cpu_to_stop)
+        cpu->exit_request = 1;
     return next_tb;
 }
 
@@ -498,19 +505,22 @@ int cpu_exec(CPUArchState *env)
                     tcg_ctx.tb_ctx.tb_invalidated_flag = 0;
                 }
 
-                AFL_QEMU_CPU_SNIPPET2;
-
                 if (qemu_loglevel_mask(CPU_LOG_EXEC)) {
                     qemu_log("Trace %p [" TARGET_FMT_lx "] %s\n",
                              tb->tc_ptr, tb->pc, lookup_symbol(tb->pc));
                 }
+/*
+ * chaining complicates AFL's instrumentation so we disable it
+ */
+#ifdef NOPE_NOT_NEVER
                 /* see if we can patch the calling TB. When the TB
                    spans two pages, we cannot safely do a direct
                    jump. */
                 if (next_tb != 0 && tb->page_addr[1] == -1) {
                     tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK),
                                 next_tb & TB_EXIT_MASK, tb);
                 }
+#endif
                 have_tb_lock = false;
                 spin_unlock(&tcg_ctx.tb_ctx.tb_lock);
 

qemu_mode/qemu/cpus.c

@@ -41,6 +41,7 @@
 #include "qemu/seqlock.h"
 #include "qapi-event.h"
 #include "hw/nmi.h"
+#include "afl.h"
 
 #ifndef _WIN32
 #include "qemu/compatfd.h"
@@ -1000,6 +1001,9 @@ static void *qemu_dummy_cpu_thread_fn(void *arg)
 
 static void tcg_exec_all(void);
 
+static int afl_qemuloop_pipe[2];        /* to notify mainloop to become forkserver */
+static CPUState *restart_cpu = NULL;    /* cpu to restart */
+
 static void *qemu_tcg_cpu_thread_fn(void *arg)
 {
     CPUState *cpu = arg;
@@ -1028,7 +1032,7 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
     /* process any pending work */
     exit_request = 1;
 
-    while (1) {
+    while (!afl_wants_cpu_to_stop) {
         tcg_exec_all();
 
         if (use_icount) {
@@ -1041,6 +1045,22 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
         qemu_tcg_wait_io_event();
     }
 
+    if(afl_wants_cpu_to_stop) {
+        /* tell iothread to run AFL forkserver */
+        afl_wants_cpu_to_stop = 0;
+        if(write(afl_qemuloop_pipe[1], "FORK", 4) != 4)
+            perror("write afl_qemuloop_pip");
+        afl_qemuloop_pipe[1] = -1;
+
+        restart_cpu = first_cpu;
+        first_cpu = NULL;
+        cpu_disable_ticks();
+
+        /* let iothread through once ... */
+        qemu_tcg_wait_io_event();
+        sleep(1);
+    }
+
     return NULL;
 }
 
@@ -1260,8 +1280,43 @@ static void qemu_dummy_start_vcpu(CPUState *cpu)
     }
 }
 
+static void
+gotPipeNotification(void *ctx)
+{
+    CPUArchState *env;
+    char buf[4];
+
+    /* cpu thread asked us to run AFL forkserver */
+    if(read(afl_qemuloop_pipe[0], buf, 4) != 4) {
+        printf("error reading afl/qemu pipe!\n");
+        exit(1);
+    }
+
+    printf("start up afl forkserver!\n");
+    afl_setup();
+    env = NULL; //XXX for now.. if we want to share JIT to the parent we will need to pass in a real env here
+    //env = restart_cpu->env_ptr;
+    afl_forkserver(env);
+
+    /* we're now in the child! */
+    tcg_cpu_thread = NULL;
+    first_cpu = restart_cpu;
+    if(aflEnableTicks) // re-enable ticks only if asked to
+        cpu_enable_ticks();
+    qemu_tcg_init_vcpu(restart_cpu);
+
+    qemu_clock_warp(QEMU_CLOCK_VIRTUAL);
+    /* continue running iothread in child process... */
+}
+
 void qemu_init_vcpu(CPUState *cpu)
 {
+    if(pipe(afl_qemuloop_pipe) == -1) {
+        perror("qemuloop pipe");
+        exit(1);
+    }
+    qemu_set_fd_handler(afl_qemuloop_pipe[0], gotPipeNotification, NULL, NULL);
+
     cpu->nr_cores = smp_cores;
     cpu->nr_threads = smp_threads;
     cpu->stopped = true;

qemu_mode/qemu/exec.c

@@ -1453,7 +1453,9 @@ static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
     if (new_block->host) {
         qemu_ram_setup_dump(new_block->host, new_block->max_length);
         qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
+        /* Keep translated memory blocks across forks for AFL!
         qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
+        */
         if (kvm_enabled()) {
             kvm_setup_guest_memory(new_block->host, new_block->max_length);
         }