[ARMIE-164] Merged latest upstream head to prep for 19.2

Minor conflict in core/arch/aarch64/codec.txt due to PR:
#3710

.travis.yml

@@ -92,9 +92,6 @@ jobs:
       compiler: clang
       # We do not have 64-bit support on OSX yet (i#1979).
       env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY=no DEPLOY=no EXTRA_ARGS=32_only
-      # XXX i#2764: Travis OSX resources are over-subscribed and it can take
-      # hours to get an OSX machine, so we skip running PR's for now.
-      if: type = push
 
 # For C/C++ there is no default install, so we set "install", not "before_install".
 install:
@@ -145,7 +142,7 @@ before_deploy:
   # We should find a way to share (xref i#1565).
   # We support setting TAG_SUFFIX on triggered builds so we can have
   # multiple unique tags in one day (the patchlevel here is the day number).
-  - export GIT_TAG="cronbuild-7.1.$((`git log -n 1 --format=%ct` / (60*60*24)))${TAG_SUFFIX}"
+  - export GIT_TAG="cronbuild-7.91.$((`git log -n 1 --format=%ct` / (60*60*24)))${TAG_SUFFIX}"
   - git tag $GIT_TAG -a -m "Travis auto-generated tag for build $TRAVIS_BUILD_NUMBER."
 deploy:
   provider: releases

CMakeLists.txt

@@ -245,7 +245,7 @@ endif (CMAKE_C_SIZEOF_DATA_PTR EQUAL 8)
 
 if (APPLE AND X64)
   # XXX #1979: 64-bit OSX is not supported.
-  message(FATAL_ERROR "64-bit Mac OSX is not supported")
+  message(WARNING "64-bit Mac OSX is not supported")
 endif ()
 
 option(VMKERNEL "target VMkernel (not officially supported yet)")
@@ -337,9 +337,12 @@ option(SET_PREFERRED_BASE "set a preferred library base address")
 if (WIN32 AND DEBUG)
   # apparently no numeric type so we use STRING
   set(preferred_base "0x15000000" CACHE STRING "Preferred library base address")
-else (WIN32 AND DEBUG)
+elseif (APPLE AND X64)
+  # Set to higher than _PAGEZERO which is [0..0x1'00000000).
+  set(preferred_base "0x171000000" CACHE STRING "Preferred library base address")
+else ()
   set(preferred_base "0x71000000" CACHE STRING "Preferred library base address")
-endif (WIN32 AND DEBUG)
+endif ()
 
 # for x64: PR 253624: we need our library to be next to our heap
 # for win32: not PIC so need a base
@@ -487,7 +490,7 @@ endif (EXISTS "${PROJECT_SOURCE_DIR}/.svn")
 
 # N.B.: when updating this, update the git tag in .travis.yml.
 # We should find a way to share (xref i#1565).
-set(VERSION_NUMBER_DEFAULT "7.1.${VERSION_NUMBER_PATCHLEVEL}")
+set(VERSION_NUMBER_DEFAULT "7.91.${VERSION_NUMBER_PATCHLEVEL}")
 # do not store the default VERSION_NUMBER in the cache to prevent a stale one
 # from preventing future version updates in a pre-existing build dir
 set(VERSION_NUMBER "" CACHE STRING "Version number: leave empty for default")
@@ -733,9 +736,10 @@ if (UNIX)
   endif ()
   if (APPLE AND CMAKE_COMPILER_IS_CLANG)
     # Ensure our binaries can run on older OSX
-    # We're building our release package on 10.11, and that XCode does not
+    # We're building our release package on 10.14, and that XCode does not
     # supply C++ libraries for below 10.9, causing drcachesim to fail to build.
     # Thus we have abandoned support for below 10.9.
+    # If this is updated, update drcachesim's as well.
     set(BASE_CFLAGS "${BASE_CFLAGS} -mmacosx-version-min=10.9")
   endif ()
   if (APPLE)
@@ -1198,11 +1202,13 @@ math(EXPR VERSION_NUMBER_INTEGER
   "${VERSION_NUMBER_MAJOR}*100 + ${VERSION_NUMBER_MINOR}")
 
 # Every release since has had minor compat breakages.
+# 7.91 broke backcompat by adding a field to instr_t .
+# 7.90 broke backcompat in DR_REG_ enums and OPSZ_ enums.
 # 6.0 broke backcompat in Linux injection, mod load event, etc.
 # 5.0 broke backcompat in drsyms and xmm opnd sizes
 # 4.1 broke backcompat in drsyms + 64-bit core (opcodes + reachability)
 # 4.0 broke backcompat in drmgr, drsyms, drinjectlib, and dr_get_milliseconds()
-set(OLDEST_COMPATIBLE_VERSION_DEFAULT "701")
+set(OLDEST_COMPATIBLE_VERSION_DEFAULT "791")
 set(OLDEST_COMPATIBLE_VERSION "" CACHE STRING
   "Oldest compatible version: leave empty for default")
 if ("${OLDEST_COMPATIBLE_VERSION}" STREQUAL "")

api/docs/intro.dox

@@ -1,5 +1,5 @@
 /* ******************************************************************************
- * Copyright (c) 2010-2018 Google, Inc.  All rights reserved.
+ * Copyright (c) 2010-2019 Google, Inc.  All rights reserved.
  * Copyright (c) 2011 Massachusetts Institute of Technology  All rights reserved.
  * Copyright (c) 2007-2010 VMware, Inc.  All rights reserved.
  * ******************************************************************************/
@@ -1070,9 +1070,10 @@ libraries) within 32-bit reachability of its code caches.  Typically, the
 code cache region is located in the low 4GB of the address space; thus, to
 avoid relocations at client library load time, it is recommended to set a
 preferred client library base in the low 4GB.  The \ref op_reachable_client
-"-reachable_client runtime option" can be used to remove this guarantee,
-though this would normally not be done unless the client is statically
-linked with the application.
+"-reachable_client runtime option" can be used to remove this guarantee.
+The option is automatically turned off when DynamoRIO and clients are
+statically linked with the application: for such static usage, 32-bit
+reachability to the code cache by clients is not guaranteed.
 
 The net result is that any static data or code in a client library, or any
 data allocated using DynamoRIO's API routines (except dr_raw_mem_alloc() or
@@ -1653,10 +1654,11 @@ Options available only in Code Manipulation mode and Memory Firewall mode
 
  - \b -reachable_client: \anchor op_reachable_client
    By default, DynamoRIO guarantees that client libraries are reachable
-   from its code caches by a 32-bit displacement.  Disabling this option
+   from its code caches by a 32-bit displacement (except for clients
+   statically linked into the application).  Disabling this option
    removes that guarantee and allows a client to be located elsewhere.
-   This would typically be used with client that is statically-linked
-   into the application.
+   The option is automatically turned off when DynamoRIO is statically
+   linked into the application.
 
  - \b -max_bb_instrs:
    DynamoRIO stops building a basic block if it hits this application

api/docs/release.dox

@@ -133,6 +133,17 @@ Dr. Memory Framework (DRMF) in the same package as DynamoRIO.  DRMF
 provides the umbra, drsyscall, and drsymcache Extensions for use by
 clients.
 
+The changes between version \DR_VERSION and 7.1.0 include the following compatibility
+changes:
+
+ - Changes the enumeration of the DR_REG_ enum by adding x86 AVX-512 registers as well
+   as reserved ranges for future extensions.
+   This is a binary compatibility change for the DR_REG_ enum.
+ - Changes the enumeration of the OPSZ_ enum by moving its start back to 0. The OPSZ_
+   enum now completely overlaps the DR_REG_ enum.
+   This is a binary compatibility change for the OPSZ_ enum.
+ - Adds a new encoding hint field to #instr_t.
+
 The changes between version \DR_VERSION and 7.1.0 include the following minor
 compatibility changes:
 
@@ -145,6 +156,21 @@ compatibility changes:
    expression and code relying on this needs to be rewritten.
    DynamoRIO_NUM_SIMD_SLOTS_COMPATIBILITY is set automatically if clients target
    version 7.1.0 or earlier.
+ - Renamed mcontext's "ymm" structure to "simd".
+ - Deprecated reg_is_xmm() and reg_is_ymm().
+ - The function reg_is_extended() now returns true for the upper 8 simd registers
+   as part of AVX-512, e.g. xmm16 - xmm31.
+ - Dropped support for clients used with statically linked DynamoRIO to reach
+   the code cache with 32-bit displacements.
+ - An additional parameter in the accessors to the drcachesim cache stats
+   (namely cache_stats_t and caching_device_stats_t) that passes a pointer to
+   the cache block being accessed (on a hit) or being replaced (on a miss).
+   This allows users to extend the cache block and stats classes in order to
+   collect more stats.
+ - Moves the immediate operand of the VEX encoded version of vextractps and vinsertps
+   to be the first source.
+ - The GPR versions of the vmovd and vmovq AVX opcodes are no longer combined into a
+   single vmovd.
 
 Further non-compatibility-affecting changes include:
 
@@ -153,6 +179,24 @@ Further non-compatibility-affecting changes include:
  - Added new fields to #dr_os_version_info_t which contain the build number,
    edition, and Windows 10 release identifier.
  - Added the function instr_is_xsave().
+ - Added the type #dr_zmm_t.
+ - Added the type #dr_opmask_t.
+ - Added the define #MCXT_NUM_OPMASK_SLOTS for the number of AVX-512 OpMask registers.
+ - Renamed mcontext's ymm structure into simd.
+ - Added a new option -logprefix to drcov.
+ - Added the AVX-512 registers #DR_REG_XMM16 - #DR_REG_XMM31, #DR_REG_YMM16 -
+   #DR_REG_YMM31 and their #DR_REG_ZMM0 - #DR_REG_ZMM31 siblings as well as
+   #DR_REG_K0 - #DR_REG_K7.
+ - Added the function reg_is_opmask().
+ - Added the functions reg_is_strictly_xmm(), reg_is_strictly_ymm() and
+   reg_is_strictly_zmm().
+ - Added the function reg_is_avx512_extended().
+ - Added the function instr_is_opmask().
+ - Added reg_set_value_ex() to set XMM, YMM and ZMM values.
+ - Added the functions instr_set_encoding_hint(), instr_has_encoding_hint().
+ - Added the type #dr_encoding_hint_type_t.
+ - Added #INSTR_ENCODING_HINT macro.
+ - Added dr_standalone_exit() with support for re-attaching afterward.
 
 **************************************************
 <hr>

clients/drcachesim/CMakeLists.txt

@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2015-2018 Google, Inc.    All rights reserved.
+# Copyright (c) 2015-2019 Google, Inc.    All rights reserved.
 # **********************************************************
 
 # Redistribution and use in source and binary forms, with or without
@@ -357,6 +357,10 @@ macro(add_win32_flags target)
     if (NOT cur MATCHES "-std=")
       append_property_string(TARGET ${target} COMPILE_FLAGS "-std=c++11")
     endif ()
+    if (APPLE)
+      # Match the core/ flags.
+      append_property_string(TARGET ${target} COMPILE_FLAGS "-mmacosx-version-min=10.9")
+    endif ()
   endif ()
 endmacro ()
 

clients/drcachesim/analyzer.cpp

@@ -76,14 +76,28 @@ ends_with(const std::string &str, const std::string &with)
 static std::unique_ptr<reader_t>
 get_reader(const std::string &path, int verbosity)
 {
-    std::unique_ptr<reader_t> reader;
 #ifdef HAS_SNAPPY
     if (ends_with(path, ".sz"))
-        reader.reset(new snappy_file_reader_t(path, verbosity));
+        return std::unique_ptr<reader_t>(new snappy_file_reader_t(path, verbosity));
+    // If path is a directory, and any file in it ends in .sz, return a snappy reader.
+    if (directory_iterator_t::is_directory(path)) {
+        directory_iterator_t end;
+        directory_iterator_t iter(path);
+        if (!iter) {
+            ERRMSG("Failed to list directory %s: %s", path.c_str(),
+                   iter.error_string().c_str());
+            return nullptr;
+        }
+        for (; iter != end; ++iter) {
+            if (ends_with(*iter, ".sz")) {
+                return std::unique_ptr<reader_t>(
+                    new snappy_file_reader_t(path, verbosity));
+            }
+        }
+    }
 #endif
-    if (reader == nullptr)
-        reader.reset(new default_file_reader_t(path, verbosity));
-    return reader;
+    // No snappy support, or didn't find a .sz file, try the default reader.
+    return std::unique_ptr<reader_t>(new default_file_reader_t(path, verbosity));
 }
 
 bool
@@ -113,8 +127,12 @@ analyzer_t::init_file_reader(const std::string &trace_path, int verbosity_in)
             if (fname == "." || fname == "..")
                 continue;
             const std::string path = trace_path + DIRSEP + fname;
+            std::unique_ptr<reader_t> reader = get_reader(path, verbosity);
+            if (!reader) {
+                return false;
+            }
             thread_data.push_back(analyzer_shard_data_t(
-                static_cast<int>(thread_data.size()), get_reader(path, verbosity), path));
+                static_cast<int>(thread_data.size()), std::move(reader), path));
             VPRINT(this, 2, "Opened reader for %s\n", path.c_str());
         }
         // Like raw2trace, we use a simple round-robin static work assigment.  This
@@ -132,6 +150,10 @@ analyzer_t::init_file_reader(const std::string &trace_path, int verbosity_in)
     } else {
         parallel = false;
         serial_trace_iter = get_reader(trace_path, verbosity);
+        if (!serial_trace_iter) {
+            return false;
+        }
+        VPRINT(this, 2, "Opened serial reader for %s\n", trace_path.c_str());
     }
     // It's ok if trace_end is a different type from serial_trace_iter, they
     // will still compare true if both at EOF.

clients/drcachesim/reader/file_reader.h

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2016-2018 Google, Inc.  All rights reserved.
+ * Copyright (c) 2016-2019 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /*
@@ -54,7 +54,7 @@
 #    ifdef WINDOWS
 #        define ZHEX64_FORMAT_STRING "%016I64x"
 #    else
-#        if defined(__i386__) || defined(__arm__)
+#        if defined(__i386__) || defined(__arm__) || defined(__APPLE__)
 #            define ZHEX64_FORMAT_STRING "%016llx"
 #        else
 #            define ZHEX64_FORMAT_STRING "%016lx"

clients/drcachesim/reader/snappy_file_reader.cpp

@@ -225,6 +225,7 @@ file_reader_t<snappy_reader_t>::open_single_file(const std::string &path)
     std::ifstream *file = new std::ifstream(path, std::ifstream::binary);
     if (!*file)
         return false;
+    VPRINT(this, 1, "Opened snappy input file %s\n", path.c_str());
     input_files.emplace_back(file);
     return true;
 }

clients/drcachesim/simulator/cache_simulator.cpp

@@ -79,6 +79,10 @@ cache_simulator_t::cache_simulator_t(const cache_simulator_knobs_t &knobs_)
         return;
     }
 
+    std::string cache_name = "LL";
+    all_caches[cache_name] = llc;
+    llcaches[cache_name] = llc;
+
     if (knobs.data_prefetcher != PREFETCH_POLICY_NEXTLINE &&
         knobs.data_prefetcher != PREFETCH_POLICY_NONE) {
         // Unknown value.
@@ -97,10 +101,6 @@ cache_simulator_t::cache_simulator_t(const cache_simulator_knobs_t &knobs_)
         return;
     }
 
-    std::string cache_name = "LL";
-    all_caches[cache_name] = llc;
-    llcaches[cache_name] = llc;
-
     l1_icaches = new cache_t *[knobs.num_cores];
     l1_dcaches = new cache_t *[knobs.num_cores];
     for (unsigned int i = 0; i < knobs.num_cores; i++) {

clients/drcachesim/simulator/cache_stats.cpp

@@ -43,7 +43,8 @@ cache_stats_t::cache_stats_t(const std::string &miss_file, bool warmup_enabled)
 }
 
 void
-cache_stats_t::access(const memref_t &memref, bool hit)
+cache_stats_t::access(const memref_t &memref, bool hit,
+                      caching_device_block_t *cache_block)
 {
     // handle prefetching requests
     if (type_is_prefetch(memref.data.type)) {
@@ -55,7 +56,7 @@ cache_stats_t::access(const memref_t &memref, bool hit)
                 dump_miss(memref);
         }
     } else { // handle regular memory accesses
-        caching_device_stats_t::access(memref, hit);
+        caching_device_stats_t::access(memref, hit, cache_block);
     }
 }
 

clients/drcachesim/simulator/cache_stats.h

@@ -47,7 +47,7 @@ class cache_stats_t : public caching_device_stats_t {
     // In addition to caching_device_stats_t::access,
     // cache_stats_t::access processes prefetching requests.
     virtual void
-    access(const memref_t &memref, bool hit);
+    access(const memref_t &memref, bool hit, caching_device_block_t *cache_block);
 
     // process CPU cache flushes
     virtual void

clients/drcachesim/simulator/caching_device.cpp

@@ -110,11 +110,12 @@ caching_device_t::request(const memref_t &memref_in)
     // Optimization: check last tag if single-block
     if (tag == final_tag && tag == last_tag) {
         // Make sure last_tag is properly in sync.
-        assert(tag != TAG_INVALID &&
-               tag == get_caching_device_block(last_block_idx, last_way).tag);
-        stats->access(memref_in, true /*hit*/);
+        caching_device_block_t *cache_block =
+            &get_caching_device_block(last_block_idx, last_way);
+        assert(tag != TAG_INVALID && tag == cache_block->tag);
+        stats->access(memref_in, true /*hit*/, cache_block);
         if (parent != NULL)
-            parent->stats->child_access(memref_in, true);
+            parent->stats->child_access(memref_in, true, cache_block);
         access_update(last_block_idx, last_way);
         return;
     }
@@ -129,36 +130,41 @@ caching_device_t::request(const memref_t &memref_in)
             memref.data.size = ((tag + 1) << block_size_bits) - memref.data.addr;
 
         for (way = 0; way < associativity; ++way) {
-            if (get_caching_device_block(block_idx, way).tag == tag) {
-                stats->access(memref, true /*hit*/);
+            caching_device_block_t *cache_block =
+                &get_caching_device_block(block_idx, way);
+            if (cache_block->tag == tag) {
+                stats->access(memref, true /*hit*/, cache_block);
                 if (parent != NULL)
-                    parent->stats->child_access(memref, true);
+                    parent->stats->child_access(memref, true, cache_block);
                 break;
             }
         }
 
         if (way == associativity) {
-            stats->access(memref, false /*miss*/);
+            way = replace_which_way(block_idx);
+            caching_device_block_t *cache_block =
+                &get_caching_device_block(block_idx, way);
+
+            stats->access(memref, false /*miss*/, cache_block);
             missed = true;
             // If no parent we assume we get the data from main memory
             if (parent != NULL) {
-                parent->stats->child_access(memref, false);
+                parent->stats->child_access(memref, false, cache_block);
                 parent->request(memref);
             }
 
             // FIXME i#1726: coherence policy
 
-            way = replace_which_way(block_idx);
             // Check if we are inserting a new block, if we are then increment
             // the block loaded count.
-            if (get_caching_device_block(block_idx, way).tag == TAG_INVALID) {
+            if (cache_block->tag == TAG_INVALID) {
                 loaded_blocks++;
             } else if (inclusive && !children.empty()) {
                 for (auto &child : children) {
-                    child->invalidate(get_caching_device_block(block_idx, way).tag);
+                    child->invalidate(cache_block->tag);
                 }
             }
-            get_caching_device_block(block_idx, way).tag = tag;
+            cache_block->tag = tag;
         }
 
         access_update(block_idx, way);