From 8e914aaa2c0464d0d50dcc6e9d0da41732e5ebbd Mon Sep 17 00:00:00 2001 From: Tim Newsham Date: Mon, 13 Jun 2016 15:58:07 -1000 Subject: [PATCH] augment QEMU to support whole-system fuzzing in AFL. --- qemu_mode/qemu/afl-qemu-cpu-inl.h | 112 +++++++++++++----- qemu_mode/qemu/afl.h | 16 +++ qemu_mode/qemu/cpu-exec.c | 14 ++- qemu_mode/qemu/cpus.c | 57 ++++++++- qemu_mode/qemu/exec.c | 2 + qemu_mode/qemu/qemu-options.hx | 8 ++ qemu_mode/qemu/target-i386/helper.h | 3 + qemu_mode/qemu/target-i386/translate.c | 157 +++++++++++++++++++++++++ qemu_mode/qemu/vl.c | 13 ++ 9 files changed, 352 insertions(+), 30 deletions(-) create mode 100644 qemu_mode/qemu/afl.h diff --git a/qemu_mode/qemu/afl-qemu-cpu-inl.h b/qemu_mode/qemu/afl-qemu-cpu-inl.h index eaa0e67b..45efbce7 100644 --- a/qemu_mode/qemu/afl-qemu-cpu-inl.h +++ b/qemu_mode/qemu/afl-qemu-cpu-inl.h @@ -27,6 +27,7 @@ */ #include +#include "afl.h" #include "../../config.h" /*************************** @@ -46,12 +47,13 @@ _start and does the usual forkserver stuff, not very different from regular instrumentation injected via afl-as.h. */ -#define AFL_QEMU_CPU_SNIPPET2 do { \ - if(tb->pc == afl_entry_point) { \ +#define AFL_QEMU_CPU_SNIPPET2(env, pc) do { \ + if(pc == afl_entry_point && pc && getenv("AFLGETWORK") == 0) { \ afl_setup(); \ afl_forkserver(env); \ + aflStart = 1; \ } \ - afl_maybe_log(tb->pc); \ + afl_maybe_log(pc); \ } while (0) /* We use one additional file descriptor to relay "needs translation" @@ -61,17 +63,27 @@ /* This is equivalent to afl-as.h: */ -static unsigned char *afl_area_ptr; +static unsigned char *afl_area_ptr = 0; /* Exported variables populated by the code patched into elfload.c: */ -abi_ulong afl_entry_point, /* ELF entry point (_start) */ - afl_start_code, /* .text start pointer */ - afl_end_code; /* .text end pointer */ +target_ulong afl_entry_point = 0, /* ELF entry point (_start) */ + afl_start_code = 0, /* .text start pointer */ + afl_end_code = 0; /* .text end pointer */ + +int aflStart = 0; /* we've started fuzzing */ +int aflEnableTicks = 0; /* re-enable ticks for each test */ +int aflGotLog = 0; /* we've seen dmesg logging */ + +/* from command line options */ +const char *aflFile = "/tmp/work"; +unsigned long aflPanicAddr = (unsigned long)-1; +unsigned long aflDmesgAddr = (unsigned long)-1; /* Set in the child process in forkserver mode: */ -static unsigned char afl_fork_child; +unsigned char afl_fork_child = 0; +int afl_wants_cpu_to_stop = 0; unsigned int afl_forksrv_pid; /* Instrumentation ratio: */ @@ -80,9 +92,7 @@ static unsigned int afl_inst_rms = MAP_SIZE; /* Function declarations. */ -static void afl_setup(void); -static void afl_forkserver(CPUArchState*); -static inline void afl_maybe_log(abi_ulong); +static inline void afl_maybe_log(target_ulong); static void afl_wait_tsl(CPUArchState*, int); static void afl_request_tsl(target_ulong, target_ulong, uint64_t); @@ -104,10 +114,9 @@ struct afl_tsl { * ACTUAL IMPLEMENTATION * *************************/ - /* Set up SHM region and initialize other stuff. */ -static void afl_setup(void) { +void afl_setup(void) { char *id_str = getenv(SHM_ENV_VAR), *inst_r = getenv("AFL_INST_RATIO"); @@ -145,16 +154,23 @@ static void afl_setup(void) { if (getenv("AFL_INST_LIBS")) { afl_start_code = 0; - afl_end_code = (abi_ulong)-1; + afl_end_code = (target_ulong)-1; } } +static ssize_t uninterrupted_read(int fd, void *buf, size_t cnt) +{ + ssize_t n; + while((n = read(fd, buf, cnt)) == -1 && errno == EINTR) + continue; + return n; +} /* Fork server logic, invoked once we hit _start. */ -static void afl_forkserver(CPUArchState *env) { +void afl_forkserver(CPUArchState *env) { static unsigned char tmp[4]; @@ -176,7 +192,7 @@ static void afl_forkserver(CPUArchState *env) { /* Whoops, parent dead? */ - if (read(FORKSRV_FD, tmp, 4) != 4) exit(2); + if (uninterrupted_read(FORKSRV_FD, tmp, 4) != 4) exit(2); /* Establish a channel with child to grab translation commands. We'll read from t_fd[0], child will write to TSL_FD. */ @@ -218,34 +234,58 @@ static void afl_forkserver(CPUArchState *env) { } - -/* The equivalent of the tuple logging routine from afl-as.h. */ - -static inline void afl_maybe_log(abi_ulong cur_loc) { - - static __thread abi_ulong prev_loc; +static inline target_ulong aflHash(target_ulong cur_loc) +{ + if(!aflStart) + return 0; /* Optimize for cur_loc > afl_end_code, which is the most likely case on Linux systems. */ if (cur_loc > afl_end_code || cur_loc < afl_start_code || !afl_area_ptr) - return; + return 0; + +#ifdef DEBUG_EDGES + if(1) { + printf("exec %lx\n", cur_loc); + fflush(stdout); + } +#endif /* Looks like QEMU always maps to fixed locations, so ASAN is not a concern. Phew. But instruction addresses may be aligned. Let's mangle the value to get something quasi-uniform. */ - cur_loc = (cur_loc >> 4) ^ (cur_loc << 8); - cur_loc &= MAP_SIZE - 1; + target_ulong h = cur_loc; + h ^= cur_loc >> 33; + h *= 0xff51afd7ed558ccd; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53; + h ^= h >> 33; + + h &= MAP_SIZE - 1; /* Implement probabilistic instrumentation by looking at scrambled block address. This keeps the instrumented locations stable across runs. */ - if (cur_loc >= afl_inst_rms) return; + if (h >= afl_inst_rms) return 0; + return h; +} + +/* todo: generate calls to helper_aflMaybeLog during translation */ +static inline void helper_aflMaybeLog(target_ulong cur_loc) { + static __thread target_ulong prev_loc; afl_area_ptr[cur_loc ^ prev_loc]++; prev_loc = cur_loc >> 1; +} + +/* The equivalent of the tuple logging routine from afl-as.h. */ +static inline void afl_maybe_log(target_ulong cur_loc) { + cur_loc = aflHash(cur_loc); + if(cur_loc) + helper_aflMaybeLog(cur_loc); } @@ -284,7 +324,25 @@ static void afl_wait_tsl(CPUArchState *env, int fd) { if (read(fd, &t, sizeof(struct afl_tsl)) != sizeof(struct afl_tsl)) break; - tb_find_slow(env, t.pc, t.cs_base, t.flags); + if(0 && env) { +#ifdef CONFIG_USER_ONLY + tb_find_slow(env, t.pc, t.cs_base, t.flags); +#else + /* if the child system emulator pages in new code and then JITs it, + and sends its address to the server, the server cannot also JIT it + without having it's guest's kernel page the data in ! + so we will only JIT kernel code segment which shouldnt page. + */ + if(t.pc >= 0xffffffff81000000 && t.pc <= 0xffffffff81ffffff) { + //printf("wait_tsl %lx -- jit\n", t.pc); fflush(stdout); + tb_find_slow(env, t.pc, t.cs_base, t.flags); + } else { + //printf("wait_tsl %lx -- ignore nonkernel\n", t.pc); fflush(stdout); + } +#endif + } else { + //printf("wait_tsl %lx -- ignore\n", t.pc); fflush(stdout); + } } diff --git a/qemu_mode/qemu/afl.h b/qemu_mode/qemu/afl.h new file mode 100644 index 00000000..f2762282 --- /dev/null +++ b/qemu_mode/qemu/afl.h @@ -0,0 +1,16 @@ + +extern const char *aflFile; +extern unsigned long aflPanicAddr; +extern unsigned long aflDmesgAddr; + +extern int aflEnableTicks; +extern int aflStart; +extern int aflGotLog; +extern target_ulong afl_start_code, afl_end_code; +extern unsigned char afl_fork_child; +extern int afl_wants_cpu_to_stop; + +void afl_setup(void); +void afl_forkserver(CPUArchState*); + + diff --git a/qemu_mode/qemu/cpu-exec.c b/qemu_mode/qemu/cpu-exec.c index 23f98c7c..f16b2172 100644 --- a/qemu_mode/qemu/cpu-exec.c +++ b/qemu_mode/qemu/cpu-exec.c @@ -198,6 +198,7 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr) #endif /* DEBUG_DISAS */ cpu->can_do_io = 0; + target_ulong pc = env->eip; next_tb = tcg_qemu_tb_exec(env, tb_ptr); cpu->can_do_io = 1; trace_exec_tb_exit((void *) (next_tb & ~TB_EXIT_MASK), @@ -216,13 +217,19 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr) assert(cc->set_pc); cc->set_pc(cpu, tb->pc); } + } else { + /* we executed it, trace it */ + AFL_QEMU_CPU_SNIPPET2(env, pc); } + if ((next_tb & TB_EXIT_MASK) == TB_EXIT_REQUESTED) { /* We were asked to stop executing TBs (probably a pending * interrupt. We've now stopped, so clear the flag. */ cpu->tcg_exit_req = 0; } + if(afl_wants_cpu_to_stop) + cpu->exit_request = 1; return next_tb; } @@ -498,12 +505,14 @@ int cpu_exec(CPUArchState *env) tcg_ctx.tb_ctx.tb_invalidated_flag = 0; } - AFL_QEMU_CPU_SNIPPET2; - if (qemu_loglevel_mask(CPU_LOG_EXEC)) { qemu_log("Trace %p [" TARGET_FMT_lx "] %s\n", tb->tc_ptr, tb->pc, lookup_symbol(tb->pc)); } +/* + * chaining complicates AFL's instrumentation so we disable it + */ +#ifdef NOPE_NOT_NEVER /* see if we can patch the calling TB. When the TB spans two pages, we cannot safely do a direct jump. */ @@ -511,6 +520,7 @@ int cpu_exec(CPUArchState *env) tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK), next_tb & TB_EXIT_MASK, tb); } +#endif have_tb_lock = false; spin_unlock(&tcg_ctx.tb_ctx.tb_lock); diff --git a/qemu_mode/qemu/cpus.c b/qemu_mode/qemu/cpus.c index e6dcae31..878718f4 100644 --- a/qemu_mode/qemu/cpus.c +++ b/qemu_mode/qemu/cpus.c @@ -41,6 +41,7 @@ #include "qemu/seqlock.h" #include "qapi-event.h" #include "hw/nmi.h" +#include "afl.h" #ifndef _WIN32 #include "qemu/compatfd.h" @@ -1000,6 +1001,9 @@ static void *qemu_dummy_cpu_thread_fn(void *arg) static void tcg_exec_all(void); +static int afl_qemuloop_pipe[2]; /* to notify mainloop to become forkserver */ +static CPUState *restart_cpu = NULL; /* cpu to restart */ + static void *qemu_tcg_cpu_thread_fn(void *arg) { CPUState *cpu = arg; @@ -1028,7 +1032,7 @@ static void *qemu_tcg_cpu_thread_fn(void *arg) /* process any pending work */ exit_request = 1; - while (1) { + while (!afl_wants_cpu_to_stop) { tcg_exec_all(); if (use_icount) { @@ -1041,6 +1045,22 @@ static void *qemu_tcg_cpu_thread_fn(void *arg) qemu_tcg_wait_io_event(); } + if(afl_wants_cpu_to_stop) { + /* tell iothread to run AFL forkserver */ + afl_wants_cpu_to_stop = 0; + if(write(afl_qemuloop_pipe[1], "FORK", 4) != 4) + perror("write afl_qemuloop_pip"); + afl_qemuloop_pipe[1] = -1; + + restart_cpu = first_cpu; + first_cpu = NULL; + cpu_disable_ticks(); + + /* let iothread through once ... */ + qemu_tcg_wait_io_event(); + sleep(1); + } + return NULL; } @@ -1260,8 +1280,43 @@ static void qemu_dummy_start_vcpu(CPUState *cpu) } } +static void +gotPipeNotification(void *ctx) +{ + CPUArchState *env; + char buf[4]; + + /* cpu thread asked us to run AFL forkserver */ + if(read(afl_qemuloop_pipe[0], buf, 4) != 4) { + printf("error reading afl/qemu pipe!\n"); + exit(1); + } + + printf("start up afl forkserver!\n"); + afl_setup(); + env = NULL; //XXX for now.. if we want to share JIT to the parent we will need to pass in a real env here + //env = restart_cpu->env_ptr; + afl_forkserver(env); + + /* we're now in the child! */ + tcg_cpu_thread = NULL; + first_cpu = restart_cpu; + if(aflEnableTicks) // re-enable ticks only if asked to + cpu_enable_ticks(); + qemu_tcg_init_vcpu(restart_cpu); + + qemu_clock_warp(QEMU_CLOCK_VIRTUAL); + /* continue running iothread in child process... */ +} + void qemu_init_vcpu(CPUState *cpu) { + if(pipe(afl_qemuloop_pipe) == -1) { + perror("qemuloop pipe"); + exit(1); + } + qemu_set_fd_handler(afl_qemuloop_pipe[0], gotPipeNotification, NULL, NULL); + cpu->nr_cores = smp_cores; cpu->nr_threads = smp_threads; cpu->stopped = true; diff --git a/qemu_mode/qemu/exec.c b/qemu_mode/qemu/exec.c index 874ecfc2..d208f38b 100644 --- a/qemu_mode/qemu/exec.c +++ b/qemu_mode/qemu/exec.c @@ -1453,7 +1453,9 @@ static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp) if (new_block->host) { qemu_ram_setup_dump(new_block->host, new_block->max_length); qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE); + /* Keep translated memory blocks across forks for AFL! qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK); + */ if (kvm_enabled()) { kvm_setup_guest_memory(new_block->host, new_block->max_length); } diff --git a/qemu_mode/qemu/qemu-options.hx b/qemu_mode/qemu/qemu-options.hx index 319d9712..02868479 100644 --- a/qemu_mode/qemu/qemu-options.hx +++ b/qemu_mode/qemu/qemu-options.hx @@ -2668,6 +2668,14 @@ STEXI @table @option ETEXI +DEF("aflFile", HAS_ARG, QEMU_OPTION_aflFile, \ + "-aflFile fname AFL input sourced from fname\n", QEMU_ARCH_ALL) +DEF("aflPanicAddr", HAS_ARG, QEMU_OPTION_aflPanicAddr, \ + "-aflPanicAddr hexaddr Address of OS panic function\n", QEMU_ARCH_ALL) +DEF("aflDmesgAddr", HAS_ARG, QEMU_OPTION_aflDmesgAddr, \ + "-aflDmesgAddr hexaddr Address of OS logging function\n", QEMU_ARCH_ALL) + + DEF("serial", HAS_ARG, QEMU_OPTION_serial, \ "-serial dev redirect the serial port to char device 'dev'\n", QEMU_ARCH_ALL) diff --git a/qemu_mode/qemu/target-i386/helper.h b/qemu_mode/qemu/target-i386/helper.h index 8eb01450..b90e98f8 100644 --- a/qemu_mode/qemu/target-i386/helper.h +++ b/qemu_mode/qemu/target-i386/helper.h @@ -1,3 +1,6 @@ +DEF_HELPER_0(aflInterceptPanic, void) +DEF_HELPER_1(aflInterceptLog, void, env) +DEF_HELPER_4(aflCall, tl, env, tl, tl, tl) DEF_HELPER_FLAGS_4(cc_compute_all, TCG_CALL_NO_RWG_SE, tl, tl, tl, tl, int) DEF_HELPER_FLAGS_4(cc_compute_c, TCG_CALL_NO_RWG_SE, tl, tl, tl, tl, int) diff --git a/qemu_mode/qemu/target-i386/translate.c b/qemu_mode/qemu/target-i386/translate.c index 305ce507..7a8a982a 100644 --- a/qemu_mode/qemu/target-i386/translate.c +++ b/qemu_mode/qemu/target-i386/translate.c @@ -128,6 +128,7 @@ typedef struct DisasContext { int cpuid_7_0_ebx_features; } DisasContext; +static void gen_aflBBlock(target_ulong pc); static void gen_eob(DisasContext *s); static void gen_jmp(DisasContext *s, target_ulong eip); static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num); @@ -7113,6 +7114,9 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s, gen_eob(s); } break; + case 0x124: /* pseudo-instr: 0x0f 0x24 - AFL call */ + gen_helper_aflCall(cpu_regs[R_EAX], cpu_env, cpu_regs[R_EDI], cpu_regs[R_ESI], cpu_regs[R_EDX]); + break; #ifdef TARGET_X86_64 case 0x105: /* syscall */ /* XXX: is it usable in real mode ? */ @@ -7992,6 +7996,7 @@ static inline void gen_intermediate_code_internal(X86CPU *cpu, cpu_cc_srcT = tcg_temp_local_new(); dc->is_jmp = DISAS_NEXT; + gen_aflBBlock(pc_start); pc_ptr = pc_start; lj = -1; num_insns = 0; @@ -8137,3 +8142,155 @@ void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, int pc_pos) if (cc_op != CC_OP_DYNAMIC) env->cc_op = cc_op; } + +#include "afl.h" + +static target_ulong startForkserver(CPUArchState *env, target_ulong enableTicks) +{ + //printf("pid %d: startForkServer\n", getpid()); fflush(stdout); + assert(!afl_fork_child); +#ifdef CONFIG_USER_ONLY + /* we're running in the main thread, get right to it! */ + afl_setup(); + afl_forkserver(env); +#else + /* + * we're running in a cpu thread. we'll exit the cpu thread + * and notify the iothread. The iothread will run the forkserver + * and in the child will restart the cpu thread which will continue + * execution. + * N.B. We assume a single cpu here! + */ + aflEnableTicks = enableTicks; + afl_wants_cpu_to_stop = 1; +#endif + return 0; +} + +/* copy work into ptr[0..sz]. Assumes memory range is locked. */ +static target_ulong getWork(CPUArchState *env, target_ulong ptr, target_ulong sz) +{ + target_ulong retsz; + FILE *fp; + unsigned char ch; + + //printf("pid %d: getWork %lx %lx\n", getpid(), ptr, sz);fflush(stdout); + assert(aflStart == 0); + fp = fopen(aflFile, "rb"); + if(!fp) { + perror(aflFile); + return -1; + } + retsz = 0; + while(retsz < sz) { + if(fread(&ch, 1, 1, fp) == 0) + break; + cpu_stb_data(env, ptr, ch); + retsz ++; + ptr ++; + } + fclose(fp); + return retsz; +} + +static target_ulong startWork(CPUArchState *env, target_ulong ptr) +{ + target_ulong start, end; + + //printf("pid %d: ptr %lx\n", getpid(), ptr);fflush(stdout); + start = cpu_ldq_data(env, ptr); + end = cpu_ldq_data(env, ptr + sizeof start); + //printf("pid %d: startWork %lx - %lx\n", getpid(), start, end);fflush(stdout); + + afl_start_code = start; + afl_end_code = end; + aflGotLog = 0; + aflStart = 1; + return 0; +} + +static target_ulong doneWork(target_ulong val) +{ + //printf("pid %d: doneWork %lx\n", getpid(), val);fflush(stdout); + assert(aflStart == 1); +/* detecting logging as crashes hasnt been helpful and + has occasionally been a problem. We'll leave it to + a post-analysis phase to look over dmesg output for + our corpus. + */ +#ifdef LETSNOT + if(aflGotLog) + exit(64 | val); +#endif + exit(val); /* exit forkserver child */ +} + +target_ulong helper_aflCall(CPUArchState *env, target_ulong code, target_ulong a0, target_ulong a1) { + switch(code) { + case 1: return startForkserver(env, a0); + case 2: return getWork(env, a0, a1); + case 3: return startWork(env, a0); + case 4: return doneWork(a0); + default: return -1; + } +} + +/* return pointer to static buf filled with strz from ptr[0..maxlen] */ +static const char * +peekStrZ(CPUArchState *env, target_ulong ptr, int maxlen) +{ + static char buf[0x1000]; + int i; + if(maxlen > sizeof buf - 1) + maxlen = sizeof buf - 1; + for(i = 0; i < maxlen; i++) { + char ch = cpu_ldub_data(env, ptr + i); + if(!ch) + break; + buf[i] = ch; + } + buf[i] = 0; + return buf; +} + +void helper_aflInterceptLog(CPUArchState *env) +{ + if(!aflStart) + return; + aflGotLog = 1; + + static FILE *fp = NULL; + if(fp == NULL) { + fp = fopen("logstore.txt", "a"); + if(fp) { + struct timeval tv; + gettimeofday(&tv, NULL); + fprintf(fp, "\n----\npid %d time %ld.%06ld\n", getpid(), (u_long)tv.tv_sec, (u_long)tv.tv_usec); + } + } + if(!fp) + return; + + target_ulong stack = env->regs[R_ESP]; + //target_ulong level = env->regs[R_ESI]; // arg 2 + target_ulong ptext = cpu_ldq_data(env, stack + 0x8); // arg7 + target_ulong len = cpu_ldq_data(env, stack + 0x10) & 0xffff; // arg8 + const char *msg = peekStrZ(env, ptext, len); + fprintf(fp, "%s\n", msg); +} + +void helper_aflInterceptPanic(void) +{ + if(!aflStart) + return; + exit(32); +} + +static void gen_aflBBlock(target_ulong pc) +{ + if(pc == aflPanicAddr) + gen_helper_aflInterceptPanic(); + if(pc == aflDmesgAddr) + gen_helper_aflInterceptLog(cpu_env); +} + diff --git a/qemu_mode/qemu/vl.c b/qemu_mode/qemu/vl.c index 74c26816..d4a65287 100644 --- a/qemu_mode/qemu/vl.c +++ b/qemu_mode/qemu/vl.c @@ -125,6 +125,10 @@ int main(int argc, char **argv) #define MAX_VIRTIO_CONSOLES 1 #define MAX_SCLP_CONSOLES 1 +extern const char *aflFile; +extern unsigned long aflPanicAddr; +extern unsigned long aflDmesgAddr; + static const char *data_dir[16]; static int data_dir_idx; const char *bios_name = NULL; @@ -3022,6 +3026,15 @@ int main(int argc, char **argv, char **envp) exit(1); } break; + case QEMU_OPTION_aflFile: + aflFile = (char *)optarg; + break; + case QEMU_OPTION_aflPanicAddr: + aflPanicAddr = strtoul(optarg, NULL, 16); + break; + case QEMU_OPTION_aflDmesgAddr: + aflDmesgAddr = strtoul(optarg, NULL, 16); + break; case QEMU_OPTION_kernel: qemu_opts_set(qemu_find_opts("machine"), 0, "kernel", optarg, &error_abort);