i#4719 qemu: Add xarch_root option for QEMU

Adds a new option -xarch_root which sets a path that is prepended to:
+ The application executable's interpreter, if the original does not exist.
+ SYS_openat paths, if the original does not exist.
+ System paths ued for loading private libraries: here the prefix is prepended
  before checking whether the original exists.

Splits dynamorio_app_init() into two pieces in order to have the
options set up at the time the loader maps the interpreter, while
avoiding ordering problems with the rest of the initialization.

The new option also auto-sets -ignore_takeover_timeout for
convenience, as that is always needed when running under QEMU.

Manually tested in cross-compile AArchXX setups on a Debian system.
Test suite integration is forthcoming.

Issue: #4719

core/dynamo.c

@@ -389,9 +389,14 @@ get_dr_stats(void)
 DYNAMORIO_EXPORT int
 dynamorio_app_init(void)
 {
-    int size;
+    dynamorio_app_init_part_one_options_and_heap();
+    return dynamorio_app_init_part_two_finalize();
+}
 
-    if (dynamo_initialized) {
+void
+dynamorio_app_init_part_one_options_and_heap(void)
+{
+    if (dynamo_initialized || dynamo_heap_initialized) {
         if (standalone_library) {
             REPORT_FATAL_ERROR_AND_EXIT(STANDALONE_ALREADY, 2, get_application_name(),
                                         get_application_pid());
@@ -479,9 +484,7 @@ dynamorio_app_init(void)
 
         /* now exit if nullcalls, now that perfctrs are set up */
         if (INTERNAL_OPTION(nullcalls)) {
-            print_file(main_logfile,
-                       "** nullcalls is set, NOT taking over execution **\n\n");
-            return SUCCESS;
+            return;
         }
 
         LOG(GLOBAL, LOG_TOP, 1, PRODUCT_NAME "'s stack size: %d Kb\n",
@@ -510,7 +513,25 @@ dynamorio_app_init(void)
 #endif
         d_r_heap_init();
         dynamo_heap_initialized = true;
+    }
+}
 
+int
+dynamorio_app_init_part_two_finalize(void)
+{
+    if (!dynamo_heap_initialized) {
+        /* Part one was never called. */
+        return FAILURE;
+    } else if (dynamo_initialized) {
+        if (standalone_library) {
+            REPORT_FATAL_ERROR_AND_EXIT(STANDALONE_ALREADY, 2, get_application_name(),
+                                        get_application_pid());
+        }
+        /* Nop. */
+    } else if (INTERNAL_OPTION(nullcalls)) {
+        print_file(main_logfile, "** nullcalls is set, NOT taking over execution **\n\n");
+        return SUCCESS;
+    } else {
         /* The process start event should be done after d_r_os_init() but before
          * process_control_int() because the former initializes event logging
          * and the latter can kill the process if a violation occurs.
@@ -618,7 +639,7 @@ dynamorio_app_init(void)
          * For now, leave it in there unless thin_client footprint becomes an
          * issue.
          */
-        size = HASHTABLE_SIZE(ALL_THREADS_HASH_BITS) * sizeof(thread_record_t *);
+        int size = HASHTABLE_SIZE(ALL_THREADS_HASH_BITS) * sizeof(thread_record_t *);
         all_threads =
             (thread_record_t **)global_heap_alloc(size HEAPACCT(ACCT_THREAD_MGT));
         memset(all_threads, 0, size);

core/globals.h

@@ -612,6 +612,11 @@ extern thread_record_t **all_threads;
 extern mutex_t all_threads_lock;
 DYNAMORIO_EXPORT int
 dynamorio_app_init(void);
+/* dynamorio_app_init() can be called in two parts: */
+void
+dynamorio_app_init_part_one_options_and_heap();
+int
+dynamorio_app_init_part_two_finalize();
 int
 dynamorio_app_exit(void);
 #if defined(CLIENT_INTERFACE) || defined(STANDALONE_UNIT_TEST)

core/optionsx.h

@@ -396,6 +396,19 @@ OPTION_DEFAULT_INTERNAL(bool, private_loader,
                                                IF_MACOS_ELSE(false, true)),
                         "use private loader for clients and dependents")
 #        ifdef UNIX
+OPTION_COMMAND_INTERNAL(pathstring_t, xarch_root, EMPTY_STRING, "xarch_root",
+                        {
+                            /* Running under QEMU requires ignoring failure to take
+                             * over QEMU threads at initialization so we bundle it
+                             * here for convenience.
+                             */
+                            if (options->xarch_root[0] != '\0') {
+                                options->ignore_takeover_timeout = true;
+                            }
+                        },
+                        "prefix to add to opened files for emulation; also sets "
+                        "-ignore_takeover_timeout",
+                        STATIC, OP_PCACHE_NOP)
 /* We cannot know the total tls size when allocating tls in os_tls_init,
  * so use the runtime option to control the tls size.
  */

core/unix/loader.c

@@ -1,5 +1,5 @@
 /* *******************************************************************************
- * Copyright (c) 2011-2020 Google, Inc.  All rights reserved.
+ * Copyright (c) 2011-2021 Google, Inc.  All rights reserved.
  * Copyright (c) 2011 Massachusetts Institute of Technology  All rights reserved.
  * *******************************************************************************/
 
@@ -887,8 +887,19 @@ privload_locate(const char *name, privmod_t *dep,
     if (dep != NULL && privload_search_rpath(dep, true /*runpath*/, name, filename))
         return true;
 
-    /* 5) FIXME: i#460, we use our system paths instead of /etc/ld.so.cache. */
+    /* 5) XXX i#460: We use our system paths instead of /etc/ld.so.cache. */
     for (i = 0; i < NUM_SYSTEM_LIB_PATHS; i++) {
+        /* First try -xarch_root for emulation. */
+        if (!IS_STRING_OPTION_EMPTY(xarch_root)) {
+            string_option_read_lock();
+            snprintf(filename, MAXIMUM_PATH, "%s/%s/%s", INTERNAL_OPTION(xarch_root),
+                     system_lib_paths[i], name);
+            filename[MAXIMUM_PATH - 1] = '\0';
+            string_option_read_unlock();
+            if (os_file_exists(filename, false /*!is_dir*/) &&
+                module_file_has_module_header(filename))
+                return true;
+        }
         snprintf(filename, MAXIMUM_PATH, "%s/%s", system_lib_paths[i], name);
         /* NULL_TERMINATE_BUFFER(filename) */
         filename[MAXIMUM_PATH - 1] = 0;
@@ -1992,11 +2003,29 @@ privload_early_inject(void **sp, byte *old_libdr_base, size_t old_libdr_size)
     reserve_brk(exe_map + exe_ld.image_size +
                 (INTERNAL_OPTION(separate_private_bss) ? PAGE_SIZE : 0));
 
+    /* Initialize DR's options and heap for the -xarch_root option. */
+    dynamorio_app_init_part_one_options_and_heap();
+
     interp = elf_loader_find_pt_interp(&exe_ld);
     if (interp != NULL) {
+        char *buf = NULL;
+        if (!IS_STRING_OPTION_EMPTY(xarch_root) && !os_file_exists(interp, false)) {
+            buf = global_heap_alloc(MAXIMUM_PATH HEAPACCT(ACCT_OTHER));
+            string_option_read_lock();
+            snprintf(buf, MAXIMUM_PATH, "%s/%s", INTERNAL_OPTION(xarch_root), interp);
+            buf[MAXIMUM_PATH - 1] = '\0';
+            string_option_read_unlock();
+            if (os_file_exists(buf, false)) {
+                LOG(GLOBAL, LOG_SYSCALLS, 2, "replacing interpreter |%s| with |%s|\n",
+                    interp, buf);
+                interp = buf;
+            }
+        }
         /* Load the ELF pointed at by PT_INTERP, usually ld.so. */
         elf_loader_t interp_ld;
         success = elf_loader_read_headers(&interp_ld, interp);
+        if (buf != NULL)
+            global_heap_free(buf, MAXIMUM_PATH HEAPACCT(ACCT_OTHER));
         apicheck(success, "Failed to read ELF interpreter headers.");
         interp_map = elf_loader_map_phdrs(
             &interp_ld, false /* fixed */, os_map_file, os_unmap_file, os_set_protection,
@@ -2021,13 +2050,15 @@ privload_early_inject(void **sp, byte *old_libdr_base, size_t old_libdr_size)
 
     elf_loader_destroy(&exe_ld);
 
-    /* Initialize DR *after* we map the app image.  This is consistent with our
-     * old behavior, and allows the client to do things like call
+    /* Initialize the rest of DR *after* we map the app and interp images.  This is
+     * consistent with our old behavior, and allows the client to do things like call
      * dr_get_proc_address() on the app from dr_client_main().  We let
      * find_executable_vm_areas re-discover the mappings we made for the app and
-     * interp images.
+     * interp images.  We do not do the full init before mapping the interp image
+     * as it complicates recording the mappings for the interp.
      */
-    dynamorio_app_init();
+    if (dynamorio_app_init_part_two_finalize() != SUCCESS)
+        apicheck(false, "Failed to initialize part two.");
 
     LOG(GLOBAL, LOG_TOP, 1, "early injected into app with this cmdline:\n");
     DOLOG(1, LOG_TOP, {

core/unix/os.c

@@ -1,5 +1,5 @@
 /* *******************************************************************************
- * Copyright (c) 2010-2020 Google, Inc.  All rights reserved.
+ * Copyright (c) 2010-2021 Google, Inc.  All rights reserved.
  * Copyright (c) 2011 Massachusetts Institute of Technology  All rights reserved.
  * Copyright (c) 2000-2010 VMware, Inc.  All rights reserved.
  * *******************************************************************************/
@@ -5024,13 +5024,24 @@ ignorable_system_call_normalized(int num)
     case SYS_epoll_pwait:
     /* Used as a lazy trigger. */
     case SYS_rseq:
+#endif
+#ifdef DEBUG
+#    ifdef MACOS
+    case SYS_open_nocancel:
+#    endif
+#    ifdef SYS_open
+    case SYS_open:
+#    endif
 #endif
         return false;
 #ifdef LINUX
 #    ifdef SYS_readlink
     case SYS_readlink:
 #    endif
     case SYS_readlinkat: return !DYNAMO_OPTION(early_inject);
+#endif
+#ifdef SYS_openat
+    case SYS_openat: return IS_STRING_OPTION_EMPTY(xarch_root);
 #endif
     default:
 #ifdef VMX86_SERVER
@@ -7707,6 +7718,33 @@ pre_system_call(dcontext_t *dcontext)
     }
 #    endif
 #endif
+#ifdef SYS_openat
+    case SYS_openat: {
+        /* XXX: For completeness we might want to replace paths for SYS_open and
+         * possibly others, but SYS_openat is all we need on modern systems so we
+         * limit syscall overhead to this single point for now.
+         */
+        dcontext->sys_param0 = 0;
+        dcontext->sys_param1 = sys_param(dcontext, 1);
+        const char *path = (const char *)dcontext->sys_param1;
+        if (!IS_STRING_OPTION_EMPTY(xarch_root) && !os_file_exists(path, false)) {
+            char *buf = heap_alloc(dcontext, MAXIMUM_PATH HEAPACCT(ACCT_OTHER));
+            string_option_read_lock();
+            snprintf(buf, MAXIMUM_PATH, "%s/%s", INTERNAL_OPTION(xarch_root), path);
+            buf[MAXIMUM_PATH - 1] = '\0';
+            string_option_read_unlock();
+            if (os_file_exists(buf, false)) {
+                LOG(THREAD, LOG_SYSCALLS, 2, "SYS_openat: replacing |%s| with |%s|\n",
+                    path, buf);
+                set_syscall_param(dcontext, 1, (reg_t)buf);
+                /* Save for freeing in post. */
+                dcontext->sys_param0 = (reg_t)buf;
+            } else
+                heap_free(dcontext, buf, MAXIMUM_PATH HEAPACCT(ACCT_OTHER));
+        }
+        break;
+    }
+#endif
 
 #ifdef LINUX
     case SYS_rseq:
@@ -8746,6 +8784,15 @@ post_system_call(dcontext_t *dcontext)
         }
         break;
 
+#    ifdef SYS_openat
+    case SYS_openat:
+        if (dcontext->sys_param0 != 0) {
+            heap_free(dcontext, (void *)dcontext->sys_param0,
+                      MAXIMUM_PATH HEAPACCT(ACCT_OTHER));
+        }
+        break;
+#    endif
+
     case SYS_rseq:
         /* Lazy rseq handling. */
         if (success) {