i#6662 public traces, part 2: encoding_filter

api/docs/release.dox

@@ -150,10 +150,10 @@ changes:
  - Changed the type of the AArch64 #dr_mcontext_t members svep and ffr to #dr_svep_t.
    This breaks binary compatibility with clients that were built against versions of
    DynamoRIO before this change.
- - Changed dynamorio::drmemtrace::record_filter_t::record_filter_func_t
+ - Changed #dynamorio::drmemtrace::record_filter_t::record_filter_func_t
    parallel_shard_filter() interface.  Added a new parameter of type
-   dynamorio::drmemtrace::record_filter_t::record_filter_info_t that allows record_filter
-   to share data with its filters.
+   #dynamorio::drmemtrace::record_filter_t::record_filter_info_t that allows
+   #dynamorio::drmemtrace::record_filter_t to share data with its filters.
 
 Further non-compatibility-affecting changes include:
  - Added DWARF-5 support to the drsyms library by linking in 4 static libraries
@@ -230,6 +230,10 @@ Further non-compatibility-affecting changes include:
    purpose of preserving register dependencies.
  - Added instr_convert_to_isa_regdeps() API that converts an #instr_t from a real ISA
    (e.g., #DR_ISA_AMD64) to the #DR_ISA_REGDEPS synthetic ISA.
+ - Added encodings2regdeps_t filter to #dynamorio::drmemtrace::record_filter_t to generate
+   #DR_ISA_REGDEPS traces.
+ - Added #dynamorio::drmemtrace::OFFLINE_FILE_TYPE_ARCH_REGDEPS file type for
+   #DR_ISA_REGDEPS traces.
 
 **************************************************
 <hr>

clients/drcachesim/CMakeLists.txt

@@ -192,13 +192,14 @@ add_exported_library(drmemtrace_simulator STATIC
   )
 
 add_exported_library(drmemtrace_record_filter STATIC
+  tools/filter/record_filter.h
   tools/filter/record_filter.cpp
   tools/filter/cache_filter.h
   tools/filter/cache_filter.cpp
   tools/filter/type_filter.h
   tools/filter/encodings2regdeps.h
   tools/filter/null_filter.h)
-target_link_libraries(drmemtrace_record_filter drmemtrace_simulator)
+target_link_libraries(drmemtrace_record_filter drmemtrace_simulator dynamorio)
 
 add_exported_library(directory_iterator STATIC common/directory_iterator.cpp)
 add_dependencies(directory_iterator api_headers)
@@ -358,6 +359,8 @@ install_client_nonDR_header(drmemtrace simulator/cache_simulator_create.h)
 install_client_nonDR_header(drmemtrace simulator/tlb_simulator_create.h)
 install_client_nonDR_header(drmemtrace tools/view_create.h)
 install_client_nonDR_header(drmemtrace tools/func_view_create.h)
+install_client_nonDR_header(drmemtrace tools/filter/record_filter_create.h)
+install_client_nonDR_header(drmemtrace tools/filter/record_filter.h)
 # TODO i#6412: Create a separate directory for non-tracer headers so that
 # we can more cleanly separate tracer and raw2trace code.
 install_client_nonDR_header(drmemtrace tracer/raw2trace.h)

clients/drcachesim/common/options.cpp

@@ -979,7 +979,7 @@ droption_t<std::string>
 droption_t<bool> op_encodings2regdeps(
     DROPTION_SCOPE_FRONTEND, "filter_encodings2regdeps", false,
     "Enable converting the encoding of instructions to synthetic ISA DR_ISA_REGDEPS.",
-    "This option is intended to be used with record_filter. When present, it converts "
+    "This option is for -simulator_type " RECORD_FILTER ". When present, it converts "
     "the encoding of instructions from a real ISA to the DR_ISA_REGDEPS synthetic ISA.");
 
 droption_t<uint64_t> op_trim_before_timestamp(

clients/drcachesim/common/trace_entry.h

@@ -668,7 +668,7 @@ type_is_instr(const trace_type_t type)
 /**
  * Returns whether \p type represents any type of instruction record whether an
  * instruction fetch or operation hint. This is a superset of type_is_instr() and includes
- * TRACE_TYPE_INSTR_NO_FETCH.
+ * #TRACE_TYPE_INSTR_NO_FETCH.
  */
 static inline bool
 is_any_instr_type(const trace_type_t type)
@@ -979,6 +979,9 @@ typedef enum {
     OFFLINE_FILE_TYPE_CORE_SHARDED = 0x10000,
     /**
      * Trace filtered by the record_filter tool using -filter_encodings2regdeps.
+     * The encodings2regdeps filter replaces real ISA encodings with #DR_ISA_REGDEPS
+     * encodings. Note that these encoding changes do not update the instruction length,
+     * hence encoding size and instruction fetch size may not match.
      */
     OFFLINE_FILE_TYPE_ARCH_REGDEPS = 0x20000,
     /**

clients/drcachesim/reader/reader.cpp

@@ -210,12 +210,12 @@ reader_t::process_input_entry()
                 ++cur_instr_count_;
             // Look for encoding bits that belong to this instr.
             if (last_encoding_.size > 0) {
-                /* OFFLINE_FILE_TYPE_ARCH_REGDEPS traces have encodings with
-                 * size != ifetch. It's a design choice, not an error, hence
-                 * we avoid this sanity check.
-                 */
-                if (!TESTANY(OFFLINE_FILE_TYPE_ARCH_REGDEPS, filetype_) &&
-                    (last_encoding_.size != cur_ref_.instr.size)) {
+                if (last_encoding_.size != cur_ref_.instr.size &&
+                    /* OFFLINE_FILE_TYPE_ARCH_REGDEPS traces have encodings with
+                     * size != ifetch. It's a design choice, not an error, hence
+                     * we avoid this sanity check for these traces.
+                     */
+                    !TESTANY(OFFLINE_FILE_TYPE_ARCH_REGDEPS, filetype_)) {
                     ERRMSG(
                         "Encoding size %zu != instr size %zu for PC 0x%zx at ord %" PRIu64
                         " instr %" PRIu64 " last_timestamp=0x%" PRIx64 "\n",

clients/drcachesim/tests/record_filter_unit_tests.cpp

@@ -321,6 +321,20 @@ test_encodings2regdeps_filter()
         { { TRACE_TYPE_ENCODING, 4, { 0xe78948 } }, true, { false } },
         { { TRACE_TYPE_ENCODING, 8, { 0x0006090600010011 } }, false, { true } },
         { { TRACE_TYPE_INSTR, 3, { 0x7f6fdd3ec360 } }, true, { true } },
+        // Same instr same chunk.
+        { { TRACE_TYPE_INSTR, 3, { 0x7f6fdd3ec360 } }, true, { true } },
+        // Unit header.
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_TIMESTAMP, { 0xc } }, true, { true } },
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CPU_ID, { 0xd } }, true, { true } },
+        // Duplicated encoding across chunk boundary.
+        { { TRACE_TYPE_ENCODING, 4, { 0xe78948 } }, true, { false } },
+        { { TRACE_TYPE_ENCODING, 8, { 0x0006090600010011 } }, false, { true } },
+        { { TRACE_TYPE_INSTR, 3, { 0x7f6fdd3ec360 } }, true, { true } },
+        // Unit header.
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_TIMESTAMP, { 0xe } }, true, { true } },
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CPU_ID, { 0xf } }, true, { true } },
+        // Same instr in different chunk.
+        { { TRACE_TYPE_INSTR, 3, { 0x7f6fdd3ec360 } }, true, { true } },
         // Trace shard footer.
         { { TRACE_TYPE_FOOTER, 0, { 0x0 } }, true, { true } },
     };

clients/drcachesim/tools/filter/encodings2regdeps.h

@@ -86,7 +86,7 @@ class encodings2regdeps_t : public record_filter_t::record_filter_func_t {
                 static_cast<trace_marker_type_t>(entry.size);
             if (marker_type == TRACE_MARKER_TYPE_FILETYPE) {
                 uint64_t marker_value = static_cast<uint64_t>(entry.addr);
-                marker_value = add_to_filetype(marker_value);
+                marker_value = update_filetype(marker_value);
                 entry.addr = static_cast<addr_t>(marker_value);
             }
         }
@@ -183,7 +183,7 @@ class encodings2regdeps_t : public record_filter_t::record_filter_func_t {
     }
 
     uint64_t
-    add_to_filetype(uint64_t filetype) override
+    update_filetype(uint64_t filetype) override
     {
         filetype &= ~OFFLINE_FILE_TYPE_ARCH_ALL;
         filetype |= OFFLINE_FILE_TYPE_ARCH_REGDEPS;

clients/drcachesim/tools/filter/record_filter.cpp

@@ -590,7 +590,7 @@ record_filter_t::process_chunk_encodings(per_shard_t *per_shard, trace_entry_t &
         // to the stop point, so a partial remove that does not change
         // the filetype?  For now we do not support that, and we re-add
         // encodings at chunk boundaries regardless. Note that filters that modify
-        // encodings (even if they add or remove trace_entry_t records) do not incurr in
+        // encodings (even if they add or remove trace_entry_t records) do not incur in
         // this problem and we don't need support for partial removal of encodings in this
         // case. An example of such filters is encodings2regdeps_t.
         if (TESTANY(OFFLINE_FILE_TYPE_ENCODINGS, per_shard->filetype) &&
@@ -651,7 +651,7 @@ record_filter_t::process_delayed_encodings(per_shard_t *per_shard, trace_entry_t
         // Output if we have encodings that haven't yet been output, and
         // there is no filter removing all encodings (we don't support
         // partial encoding removal). Note that filters that modify encodings (even if
-        // they add or remove trace_entry_t records) do not incurr in this problem and we
+        // they add or remove trace_entry_t records) do not incur in this problem and we
         // don't need support for partial removal of encodings in this case. An example
         // of such filters is encodings2regdeps_t.
         // We check prev_was_output to rule out filtered-out encodings

clients/drcachesim/tools/filter/record_filter.h

@@ -65,6 +65,10 @@ class record_filter_t : public record_analysis_tool_t {
      * Interface for the record_filter to share data with its filters.
      */
     struct record_filter_info_t {
+        /**
+         * Stores the encoding of an instructions, which may be split among more than one
+         * #trace_entry_t, hence the vector.
+         */
         std::vector<trace_entry_t> *last_encoding;
     };
 
@@ -93,10 +97,10 @@ class record_filter_t : public record_analysis_tool_t {
         /**
          * Invoked for each #trace_entry_t in the shard. It returns
          * whether or not this \p entry should be included in the result
-         * trace. \p shard_data is same as what was returned by
-         * parallel_shard_init(). The given \p entry is included in the result
-         * trace iff all provided #record_filter_func_t return true. The
-         * \p entry parameter can also be modified by the record_filter_func_t.
+         * trace. \p shard_data is same as what was returned by parallel_shard_init().
+         * The given \p entry is included in the result trace iff all provided
+         * #dynamorio::drmemtrace::record_filter_t::record_filter_func_t return true.
+         * The \p entry parameter can also be modified by the record_filter_func_t.
          * The passed \p entry is not guaranteed to be the original one from
          * the trace if other filter tools are present, and may include changes
          * made by other tools.
@@ -127,10 +131,10 @@ class record_filter_t : public record_analysis_tool_t {
         /**
          * If a filter modifies the file type of a trace, its changes should be made here,
          * so they are visible to the record_filter even if the #trace_entry_t containing
-         * the file type marker is not modified.
+         * the file type marker is not modified directly by the filter.
          */
         virtual uint64_t
-        add_to_filetype(uint64_t filetype)
+        update_filetype(uint64_t filetype)
         {
             return filetype;
         }
@@ -277,7 +281,7 @@ class record_filter_t : public record_analysis_tool_t {
         /* If filters modify the file type, add their changes here.
          */
         for (auto &filter : filters_) {
-            filetype = filter->add_to_filetype(filetype);
+            filetype = filter->update_filetype(filetype);
         }
         return filetype;
     }

clients/drcachesim/tools/filter/record_filter_create.h

@@ -62,8 +62,9 @@ namespace drmemtrace {
  *   up to its first timestamp whose value is greater or equal to this parameter.
  * @param[in] trim_after_timestamp  Trim records after the trace's first timestamp
  *   whose value is greater than this parameter.
- * @param[in] encodings2regdeps If true, converts instruction encodings from the real ISA
+ * @param[in] encodings2regdeps  If true, converts instruction encodings from the real ISA
  *   of the input trace to the #DR_ISA_REGDEPS synthetic ISA.
+ * @param[in] verbose  Verbosity level for notifications.
  */
 record_analysis_tool_t *
 record_filter_tool_create(const std::string &output_dir, uint64_t stop_timestamp,

clients/drcachesim/tools/record_filter_launcher.cpp

@@ -128,7 +128,7 @@ static droption_t<uint64_t> op_trim_after_timestamp(
 droption_t<bool> op_encodings2regdeps(
     DROPTION_SCOPE_FRONTEND, "filter_encodings2regdeps", false,
     "Enable converting the encoding of instructions to synthetic ISA DR_ISA_REGDEPS.",
-    "This option is intended to be used with record_filter. When present, it converts "
+    "This option is for -simulator_type record_filter. When present, it converts "
     "the encoding of instructions from a real ISA to the DR_ISA_REGDEPS synthetic ISA.");
 } // namespace
 

core/ir/x86/decode.c

@@ -2791,6 +2791,13 @@ const char *
 decode_opcode_name(int opcode)
 {
     const instr_info_t *info = op_instr[opcode];
+    if (info == NULL) {
+        switch (opcode) {
+        case OP_INVALID: return "<invalid>";
+        case OP_UNDECODED: return "<undecoded>";
+        default: return "<unknown>";
+        }
+    }
     return info->name;
 }
 

core/ir/x86/decode_table.c

@@ -75,7 +75,7 @@
 const instr_info_t * const op_instr[] =
 {
     /* OP_INVALID */   NULL,
-    /* OP_UNDECODED */ (instr_info_t[]){{1, 0xffffff, 0, "<undecoded>", 0, 0, 0, 0, 0, 0, 0, 0}},
+    /* OP_UNDECODED */ NULL,
     /* OP_CONTD   */   NULL,
     /* OP_LABEL   */   NULL,