i#909 early execve: implements early injection across execve

Previously, even if the parent was invoked with -early_inject, any
subsequent execve was injected into using LD_PRELOAD.
This CL implements early injection across execve by substituting the
libdynamorio.so path for the app's target path (which requires adding new
filepath buffers).  We also replace its argv[0] if it's /proc/self/exe.

It turns out that our unsetenv during init shifts the env entries and
messes up ld.so's access to auxv.  We have to change those calls to a new
routine disable_env() which leaves the entries in place but changes their
names.

The linux.readlink test functions as a regression test for this.

Fixes #909

Review-URL: https://codereview.appspot.com/213290043

core/dynamo.c

@@ -387,7 +387,8 @@ dynamorio_app_init(void)
             os_tls_pre_init(atoi(getenv(DYNAMORIO_VAR_EXECVE)));
 # endif
             /* important to remove it, don't want to propagate to forked children, etc. */
-            unsetenv(DYNAMORIO_VAR_EXECVE);
+            /* i#909: unsetenv is unsafe as it messes up auxv access, so we disable */
+            disable_env(DYNAMORIO_VAR_EXECVE);
             /* check that it's gone: we've had problems with unsetenv */
             ASSERT(getenv(DYNAMORIO_VAR_EXECVE) == NULL);
         } else

core/unix/loader.c

@@ -1568,6 +1568,15 @@ privload_early_inject(void **sp)
      */
     dynamorio_app_init();
 
+    LOG(GLOBAL, LOG_TOP, 1, "early injected into app with this cmdline:\n");
+    DOLOG(1, LOG_TOP, {
+        int i;
+        for (i = 0; i < *argc; i++) {
+            LOG(GLOBAL, LOG_TOP, 1, "%s ", argv[i]);
+        }
+        LOG(GLOBAL, LOG_TOP, 1, "\n");
+    });
+
     if (RUNNING_WITHOUT_CODE_CACHE()) {
         /* Reset the stack pointer back to the beginning and jump to the entry
          * point to execute the app natively.  This is also useful for testing

core/unix/module.c

@@ -723,13 +723,17 @@ at_dl_runtime_resolve_ret(dcontext_t *dcontext, app_pc source_fragment, int *ret
 }
 
 bool
-module_file_is_module64(file_t f)
+module_file_is_module64(file_t f, bool *is64 OUT)
 {
     dr_platform_t platform;
-    if (module_get_platform(f, &platform))
-        return platform == DR_PLATFORM_64BIT;
-    /* on error, assume same arch as us */
-    return IF_X64_ELSE(true, false);
+    if (module_get_platform(f, &platform)) {
+        if (platform == DR_PLATFORM_64BIT)
+            *is64 = true;
+        else
+            *is64 = false;
+        return true;
+    }
+    return false;
 }
 
 bool

core/unix/module.h

@@ -131,7 +131,7 @@ bool
 module_file_has_module_header(const char *filename);
 
 bool
-module_file_is_module64(file_t f);
+module_file_is_module64(file_t f, bool *is64 OUT);
 
 bool
 module_get_platform(file_t f, dr_platform_t *platform);

core/unix/os.c

@@ -257,9 +257,11 @@ static void handle_app_brk(dcontext_t *dcontext, byte *old_brk, byte *new_brk);
 #endif
 
 /* full path to our own library, used for execve */
-static char dynamorio_library_path[MAXIMUM_PATH];
+static char dynamorio_library_path[MAXIMUM_PATH]; /* just dir */
+static char dynamorio_library_filepath[MAXIMUM_PATH];
 /* Issue 20: path to other architecture */
 static char dynamorio_alt_arch_path[MAXIMUM_PATH];
+static char dynamorio_alt_arch_filepath[MAXIMUM_PATH]; /* just dir */
 /* Makefile passes us LIBDIR_X{86,64} defines */
 #define DR_LIBDIR_X86 STRINGIFY(LIBDIR_X86)
 #define DR_LIBDIR_X64 STRINGIFY(LIBDIR_X64)
@@ -557,6 +559,33 @@ our_unsetenv(const char *name)
     return 0;
 }
 
+/* Clobbers the name rather than shifting, to preserve auxv (xref i#909). */
+bool
+disable_env(const char *name)
+{
+    size_t name_len;
+    char **env = our_environ;
+    if (name == NULL || *name == '\0' || strchr(name, '=') != NULL) {
+        return false;
+    }
+    ASSERT(our_environ != NULL);
+    if (our_environ == NULL)
+        return false;
+    name_len = strlen(name);
+    while (*env != NULL) {
+        if (strncmp(*env, name, name_len) == 0 && (*env)[name_len] == '=') {
+            /* We have a match.  If we shift subsequent entries we'll mess
+             * up access to auxv, which is after the env block, so we instead
+             * disable the env var by changing its name.
+             * We keep going to handle later matches.
+             */
+            snprintf(*env, name_len, "__disabled__");
+        }
+        env++;
+    }
+    return true;
+}
+
 /* i#46: Private getenv.
  */
 char *
@@ -4859,6 +4888,7 @@ enum {
     ENV_PROP_RUNUNDER,
     ENV_PROP_OPTIONS,
     ENV_PROP_EXECVE_LOGDIR,
+    ENV_PROP_EXE_PATH,
 };
 
 static const char * const env_to_propagate[] = {
@@ -4871,14 +4901,16 @@ static const char * const env_to_propagate[] = {
      * Xref comment in create_log_dir about their precedence.
      */
     DYNAMORIO_VAR_EXECVE_LOGDIR,
+    /* i#909: needed for early injection */
+    DYNAMORIO_VAR_EXE_PATH,
     /* these will only be propagated if they exist */
     DYNAMORIO_VAR_CONFIGDIR,
 };
 #define NUM_ENV_TO_PROPAGATE (sizeof(env_to_propagate)/sizeof(env_to_propagate[0]))
 
 /* called at pre-SYS_execve to append DR vars in the target process env vars list */
 static void
-add_dr_env_vars(dcontext_t *dcontext, char *inject_library_path)
+add_dr_env_vars(dcontext_t *dcontext, char *inject_library_path, const char *app_path)
 {
     char **envp = (char **) sys_param(dcontext, 2);
     int idx, j, preload = -1, ldpath = -1;
@@ -4904,6 +4936,9 @@ add_dr_env_vars(dcontext_t *dcontext, char *inject_library_path)
     else
         need_var[ENV_PROP_EXECVE_LOGDIR] = false;
 
+    if (DYNAMO_OPTION(early_inject))
+        need_var[ENV_PROP_EXE_PATH] = true;
+
     /* iterate the env in target process  */
     if (envp == NULL) {
         LOG(THREAD, LOG_SYSCALLS, 3, "\tenv is NULL\n");
@@ -4921,12 +4956,14 @@ add_dr_env_vars(dcontext_t *dcontext, char *inject_library_path)
                     break;
                 }
             }
-            if (strstr(envp[idx], "LD_LIBRARY_PATH=") == envp[idx]) {
+            if (!DYNAMO_OPTION(early_inject) &&
+                strstr(envp[idx], "LD_LIBRARY_PATH=") == envp[idx]) {
                 ldpath = idx;
                 if (strstr(envp[idx], inject_library_path) != NULL)
                     ldpath_us = true;
             }
-            if (strstr(envp[idx], "LD_PRELOAD=") == envp[idx]) {
+            if (!DYNAMO_OPTION(early_inject) &&
+                strstr(envp[idx], "LD_PRELOAD=") == envp[idx]) {
                 preload = idx;
                 if (strstr(envp[idx], DYNAMORIO_PRELOAD_NAME) != NULL &&
                     strstr(envp[idx], DYNAMORIO_LIBRARY_NAME) != NULL) {
@@ -4946,8 +4983,9 @@ add_dr_env_vars(dcontext_t *dcontext, char *inject_library_path)
     /* how many new env vars we need add */
     num_new =
         2 + /* execve indicator var plus final NULL */
-        ((preload<0) ? 1 : 0) +
-        ((ldpath<0) ? 1 : 0);
+        (DYNAMO_OPTION(early_inject) ? 0 :
+         (((preload<0) ? 1 : 0) +
+          ((ldpath<0) ? 1 : 0)));
 
     if (DYNAMO_OPTION(follow_children)) {
         for (j = 0; j < NUM_ENV_TO_PROPAGATE; j++) {
@@ -4961,7 +4999,7 @@ add_dr_env_vars(dcontext_t *dcontext, char *inject_library_path)
     /* copy old envp */
     memcpy(new_envp, envp, sizeof(char*)*num_old);
     /* change/add preload and ldpath if necessary */
-    if (!preload_us) {
+    if (!DYNAMO_OPTION(early_inject) && !preload_us) {
         int idx_preload;
         LOG(THREAD, LOG_SYSCALLS, 1,
             "WARNING: execve env does NOT preload DynamoRIO, forcing it!\n");
@@ -4988,7 +5026,7 @@ add_dr_env_vars(dcontext_t *dcontext, char *inject_library_path)
         LOG(THREAD, LOG_SYSCALLS, 2, "\tnew env %d: %s\n",
             idx_preload, new_envp[idx_preload]);
     }
-    if (!ldpath_us) {
+    if (!DYNAMO_OPTION(early_inject) && !ldpath_us) {
         int idx_ldpath;
         if (ldpath >= 0) {
             sz = strlen(envp[ldpath]) + strlen(inject_library_path) + 2;
@@ -5031,6 +5069,10 @@ add_dr_env_vars(dcontext_t *dcontext, char *inject_library_path)
                 ASSERT(strcmp(env_to_propagate[j], DYNAMORIO_VAR_EXECVE_LOGDIR) == 0);
                 ASSERT(get_log_dir(PROCESS_DIR, NULL, NULL));
                 break;
+            case ENV_PROP_EXE_PATH:
+                ASSERT(strcmp(env_to_propagate[j], DYNAMORIO_VAR_EXE_PATH) == 0);
+                val = app_path;
+                break;
             default:
                 val = getenv(env_to_propagate[j]);
                 if (val == NULL)
@@ -5119,19 +5161,17 @@ handle_execve(dcontext_t *dcontext)
      */
     char *fname = (char *) sys_param(dcontext, 0);
     bool x64 = IF_X64_ELSE(true, false);
+    bool expect_to_fail = false;
     file_t file;
     char *inject_library_path;
-    DEBUG_DECLARE(char **argv = (char **) sys_param(dcontext, 1);)
+    const char **argv = (const char **) sys_param(dcontext, 1);
 #ifdef LINUX
     if (DYNAMO_OPTION(early_inject) && symlink_is_self_exe(fname)) {
         /* i#907: /proc/self/exe points at libdynamorio.so.  Make sure we run
          * the right thing here.
          */
-        dcontext->sys_param4 = (reg_t) fname; /* store for restore in post */
         fname = get_application_name();
-        *sys_param_addr(dcontext, 0) = (reg_t) get_application_name();
-    } else
-        dcontext->sys_param4 = 0; /* no restore in post */
+    }
 #endif
 
     LOG(GLOBAL, LOG_ALL, 1, "\n---------------------------------------------------------------------------\n");
@@ -5185,13 +5225,45 @@ handle_execve(dcontext_t *dcontext)
      */
     file = os_open(fname, OS_OPEN_READ);
     if (file != INVALID_FILE) {
-        x64 = module_file_is_module64(file);
+        if (!module_file_is_module64(file, &x64))
+            expect_to_fail = true;
         os_close(file);
-    }
+    } else
+        expect_to_fail = true;
     inject_library_path = IF_X64_ELSE(x64, !x64) ? dynamorio_library_path :
         dynamorio_alt_arch_path;
 
-    add_dr_env_vars(dcontext, inject_library_path);
+    add_dr_env_vars(dcontext, inject_library_path, fname);
+
+#ifdef LINUX
+    /* We have to be accurate with expect_to_fail as we cannot come back
+     * and fail the syscall once the kernel execs DR!
+     */
+    if (DYNAMO_OPTION(early_inject) && !expect_to_fail) {
+        /* i#909: change the target image to libdynamorio.so */
+        const char *drpath = IF_X64_ELSE(x64, !x64) ? dynamorio_library_filepath :
+            dynamorio_alt_arch_filepath;
+        TRY_EXCEPT(dcontext, /* try */ {
+            if (symlink_is_self_exe(argv[0])) {
+                /* we're out of sys_param entries so we assume argv[0] == fname */
+                dcontext->sys_param3 = (reg_t) argv;
+                argv[0] = fname; /* XXX: handle readable but not writable! */
+            } else
+                dcontext->sys_param3 = 0; /* no restore in post */
+            dcontext->sys_param4 = (reg_t) fname; /* store for restore in post */
+            *sys_param_addr(dcontext, 0) = (reg_t) drpath;
+            LOG(THREAD, LOG_SYSCALLS, 2, "actual execve on: %s\n",
+                (char *)sys_param(dcontext, 0));
+        }, /* except */ {
+            dcontext->sys_param3 = 0; /* no restore in post */
+            dcontext->sys_param4 = 0; /* no restore in post */
+            LOG(THREAD, LOG_SYSCALLS, 2, "argv is unreadable, expect execve to fail\n");
+        });
+    } else {
+        dcontext->sys_param3 = 0; /* no restore in post */
+        dcontext->sys_param4 = 0; /* no restore in post */
+    }
+#endif
 
     /* we need to clean up the .1config file here.  if the execve fails,
      * we'll just live w/o dynamic option re-read.
@@ -5217,6 +5289,11 @@ handle_execve_post(dcontext_t *dcontext)
     if (dcontext->sys_param4 != 0) {
         /* restore original /proc/.../exe */
         *sys_param_addr(dcontext, 0) = dcontext->sys_param4;
+        if (dcontext->sys_param3 != 0) {
+            /* restore original argv[0] */
+            const char **argv = (const char **) dcontext->sys_param3;
+            argv[0] = (const char *) dcontext->sys_param4;
+        }
     }
 #endif
     if (new_envp != NULL) {
@@ -7494,6 +7571,9 @@ get_dynamo_library_bounds(void)
                                   BUFFER_SIZE_ELEMENTS(dynamorio_library_path));
     LOG(GLOBAL, LOG_VMAREAS, 1, PRODUCT_NAME" library path: %s\n",
         dynamorio_library_path);
+    snprintf(dynamorio_library_filepath, BUFFER_SIZE_ELEMENTS(dynamorio_library_filepath),
+             "%s%s", dynamorio_library_path, dynamorio_libname);
+    NULL_TERMINATE_BUFFER(dynamorio_library_filepath);
 #if !defined(STATIC_LIBRARY) && defined(LINUX)
     ASSERT(check_start == dynamo_dll_start && check_end == dynamo_dll_end);
 #elif defined(MACOS)
@@ -7522,6 +7602,10 @@ get_dynamo_library_bounds(void)
     NULL_TERMINATE_BUFFER(dynamorio_alt_arch_path);
     LOG(GLOBAL, LOG_VMAREAS, 1, PRODUCT_NAME" alt arch path: %s\n",
         dynamorio_alt_arch_path);
+    snprintf(dynamorio_alt_arch_filepath,
+             BUFFER_SIZE_ELEMENTS(dynamorio_alt_arch_filepath),
+             "%s%s", dynamorio_alt_arch_path, dynamorio_libname);
+    NULL_TERMINATE_BUFFER(dynamorio_alt_arch_filepath);
 
     return res;
 }

core/unix/os_exports.h

@@ -177,7 +177,11 @@ char *our_getenv(const char *name);
 
 /* to avoid unsetenv problems we have our own unsetenv */
 #define unsetenv our_unsetenv
+/* XXX: unsetenv is unsafe to call during init, as it messes up access to auxv!
+ * Use disable_env instead.  Xref i#909.
+ */
 int our_unsetenv(const char *name);
+bool disable_env(const char *name);
 
 /* new segment support
  * name is a string

core/utils.c

@@ -2656,7 +2656,8 @@ create_log_dir(int dir_type)
             logdir_initialized = true;
         }
         /* important to remove it, don't want to propagate to forked children */
-        unsetenv(DYNAMORIO_VAR_EXECVE_LOGDIR);
+        /* i#909: unsetenv is unsafe as it messes up auxv access, so we disable */
+        disable_env(DYNAMORIO_VAR_EXECVE_LOGDIR);
         /* check that it's gone: we've had problems with unsetenv */
         ASSERT(getenv(DYNAMORIO_VAR_EXECVE_LOGDIR) == NULL);
     }