traffic_dump: add tls information to dump. (#6727)

This change adds tls information nodes like the following:

    "tls": {
	"sni": "<SNI>",
	"verify_mode": "<SSL_VERIFY_MODE_VALUE>"
    },

Co-authored-by: bneradt <bneradt@verizonmedia.com>
(cherry picked from commit bfafd91)

Author: 2 people

plugins/experimental/traffic_dump/traffic_dump.cc

@@ -75,9 +75,9 @@ struct StringHashByLower {
 };
 
 /// Fields considered sensitive because they may contain user-private
-/// information. These fields are replaced with auto-generated generic content
-/// by default. To turn off this behavior, the user should add the
-/// --promiscuous-mode flag as a commandline argument.
+/// information. These fields are replaced with auto-generated generic content by
+/// default. To override this behavior, the user should specify their own fields
+/// they consider sensitive with --sensitive-fields.
 ///
 /// While these are specified with case, they are matched case-insensitively.
 std::unordered_set<std::string, StringHashByLower, InsensitiveCompare> default_sensitive_fields = {
@@ -568,6 +568,143 @@ session_txn_handler(TSCont contp, TSEvent event, void *edata)
   return TS_SUCCESS;
 }
 
+/** Create a TLS characteristics node.
+ *
+ * This function encapsulates the logic common between the client-side and
+ * server-side logic for populating the TLS node.
+ *
+ * @param[in] ssnp The pointer for this session.
+ *
+ * @return The node describing the TLS properties of this session.
+ */
+std::string
+get_tls_description_helper(TSVConn ssn_vc)
+{
+  TSSslConnection ssl_conn = TSVConnSslConnectionGet(ssn_vc);
+  SSL *ssl_obj             = (SSL *)ssl_conn;
+  if (ssl_obj == nullptr) {
+    return "";
+  }
+  std::ostringstream tls_description;
+  tls_description << R"("tls":{)";
+  const char *sni_ptr = SSL_get_servername(ssl_obj, TLSEXT_NAMETYPE_host_name);
+  if (sni_ptr != nullptr) {
+    std::string_view sni{sni_ptr};
+    if (!sni.empty()) {
+      tls_description << R"("sni":")" << sni << R"(")";
+    }
+  }
+  tls_description << R"(,"verify_mode":")" << std::to_string(SSL_get_verify_mode(ssl_obj)) << R"(")";
+  tls_description << "}";
+  return tls_description.str();
+}
+
+/** Create a server-side TLS characteristics node.
+ *
+ * @param[in] ssnp The pointer for this session.
+ *
+ * @return The node describing the TLS properties of this session.
+ */
+std::string
+get_server_tls_description(TSHttpSsn ssnp)
+{
+  TSVConn ssn_vc = TSHttpSsnServerVConnGet(ssnp);
+  return get_tls_description_helper(ssn_vc);
+}
+
+/** Create a client-side TLS characteristics node.
+ *
+ * @param[in] ssnp The pointer for this session.
+ *
+ * @return The node describing the TLS properties of this session.
+ */
+std::string
+get_client_tls_description(TSHttpSsn ssnp)
+{
+  TSVConn ssn_vc = TSHttpSsnClientVConnGet(ssnp);
+  return get_tls_description_helper(ssn_vc);
+}
+
+/// A named boolean for callers who pass the is_client parameter.
+constexpr bool IS_CLIENT = true;
+
+/** Create the nodes that describe the session's sub-HTTP protocols.
+ *
+ * This function encapsulates the logic common between the client-side and
+ * server-side logic for describing the session's characteristics.
+ *
+ * This will create the string representing the "protocol" and "tls" nodes. The
+ * "tls" node will only be present if the connection is over SSL/TLS.
+ *
+ * @param[in] ssnp The pointer for this session.
+ *
+ * @return The description of the protocol stack and certain TLS attributes.
+ */
+std::string
+get_protocol_description_helper(TSHttpSsn ssnp, bool is_client)
+{
+  std::ostringstream protocol_description;
+  protocol_description << R"("protocol":[)";
+
+  const char *protocol[10];
+  int count = -1;
+  if (is_client) {
+    TSAssert(TS_SUCCESS == TSHttpSsnClientProtocolStackGet(ssnp, 10, protocol, &count));
+  } else {
+    // See the TODO below in the commented out defintion of get_server_protocol_description.
+    // TSAssert(TS_SUCCESS == TSHttpSsnServerProtocolStackGet(ssnp, 10, protocol, &count));
+  }
+  for (int i = 0; i < count; i++) {
+    if (i > 0) {
+      protocol_description << ",";
+    }
+    protocol_description << '"' << std::string(protocol[i]) << '"';
+  }
+
+  protocol_description << "]";
+  std::string tls_description;
+  if (is_client) {
+    tls_description = get_client_tls_description(ssnp);
+  } else {
+    tls_description = get_server_tls_description(ssnp);
+  }
+  if (!tls_description.empty()) {
+    protocol_description << "," << tls_description;
+  }
+  return protocol_description.str();
+}
+
+#if 0
+// TODO It will be important to add this eventually, but
+// TSHttpSsnServerProtocolStackGet is not defined yet. Once it (or some other
+// mechanism for getting the server side stack) is implemented, we will call
+// this as a part of writing the server-response node.
+
+/** Generate the nodes describing the server session.
+ *
+ * @param[in] ssnp The pointer for this session.
+ *
+ * @return The description of the protocol stack and certain TLS attributes.
+ */
+std::string
+get_server_protocol_description(TSHttpSsn ssnp)
+{
+  return get_protocol_description_helper(ssnp, !IS_CLIENT);
+}
+#endif
+
+/** Generate the nodes describing the client session.
+ *
+ * @param[in] ssnp The pointer for this session.
+ *
+ * @return The description of the protocol stack and certain TLS attributes.
+ */
+std::string
+get_client_protocol_description(TSHttpSsn ssnp)
+{
+  return get_protocol_description_helper(ssnp, IS_CLIENT);
+}
+
 // Session handler for global hooks; Assign per-session data structure and log files
 static int
 global_ssn_handler(TSCont contp, TSEvent event, void *edata)
@@ -621,9 +758,9 @@ global_ssn_handler(TSCont contp, TSEvent event, void *edata)
         TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore HTTPS session with non-existent SNI.");
         break;
       } else {
-        const std::string sni{sni_ptr};
+        const std::string_view sni{sni_ptr};
         if (sni != sni_filter) {
-          TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore HTTPS session with non-filtered SNI: %s", sni.c_str());
+          TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore HTTPS session with non-filtered SNI: %s", sni_ptr);
           break;
         }
       }
@@ -647,19 +784,11 @@ global_ssn_handler(TSCont contp, TSEvent event, void *edata)
 
     TSContDataSet(ssnData->aio_cont, ssnData);
 
-    // 1. "protocol":(string)
-    const char *protocol[10];
-    int count = 0;
-    TSAssert(TS_SUCCESS == TSHttpSsnClientProtocolStackGet(ssnp, 10, protocol, &count));
-    std::string result;
-    for (int i = 0; i < count; i++) {
-      if (i > 0) {
-        result += ",";
-      }
-      result += '"' + std::string(protocol[i]) + '"';
-    }
+    // "protocol":(string),"tls":(string)
+    // The "tls" node will only be present if the session is over SSL/TLS.
+    std::string protocol_description = get_client_protocol_description(ssnp);
 
-    std::string beginning = R"({"meta":{"version":"1.0"},"sessions":[{"protocol":[)" + result + "]" + R"(,"connection-time":)" +
+    std::string beginning = R"({"meta":{"version":"1.0"},"sessions":[{)" + protocol_description + R"(,"connection-time":)" +
                             std::to_string(start.count()) + R"(,"transactions":[)";
 
     // Use the session count's hex string as the filename.

tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py

@@ -103,13 +103,14 @@
 tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port))
 tr.Processes.Default.StartBefore(Test.Processes.ts)
 tr.Processes.Default.Command = \
-        ('curl http://127.0.0.1:{0} -H"Cookie: donotlogthis" '
+        ('curl --http1.1 http://127.0.0.1:{0} -H"Cookie: donotlogthis" '
          '-H"Host: www.example.com" -H"X-Request-1: ultra_sensitive" --verbose'.format(
              ts.Variables.port))
 tr.Processes.Default.ReturnCode = 0
 tr.Processes.Default.Streams.stderr = "gold/200.gold"
 tr.StillRunningAfter = server
 tr.StillRunningAfter = ts
+session_1_protocols = "tcp,ipv4"
 
 # Execute the second transaction.
 tr = Test.AddTestRun("Second transaction")
@@ -131,11 +132,12 @@
         "--sensitive-fields x-request-1 "
         "--sensitive-fields x-request-2 ")
 tr.Setup.CopyAs(verify_replay, Test.RunDirectory)
-tr.Processes.Default.Command = "python3 {0} {1} {2} {3}".format(
+tr.Processes.Default.Command = 'python3 {0} {1} {2} {3} --client-protocols "{4}"'.format(
         verify_replay,
         os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'),
         replay_file_session_1,
-        sensitive_fields_arg)
+        sensitive_fields_arg,
+        session_1_protocols)
 tr.Processes.Default.ReturnCode = 0
 tr.StillRunningAfter = server
 tr.StillRunningAfter = ts

tests/gold_tests/pluginTest/traffic_dump/traffic_dump_sni_filter.test.py

@@ -119,12 +119,14 @@
 tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port))
 tr.Processes.Default.StartBefore(Test.Processes.ts)
 tr.Processes.Default.Command = \
-        ('curl --tls-max 1.2 -k -H"Host: bob" --resolve "bob:{0}:127.0.0.1" '
+        ('curl --http2 --tls-max 1.2 -k -H"Host: bob" --resolve "bob:{0}:127.0.0.1" '
          '--cert ./signed-foo.pem --key ./signed-foo.key --verbose https://bob:{0}'.format(ts.Variables.ssl_port))
 tr.Processes.Default.ReturnCode = 0
 tr.Processes.Default.Streams.stderr = "gold/200_sni_bob.gold"
 tr.StillRunningAfter = server
 tr.StillRunningAfter = ts
+session_1_protocols = "h2,tls/1.2,tcp,ipv4"
+session_1_tls_features = 'sni:bob'
 
 # Execute the second transaction with an SNI of dave.
 tr = Test.AddTestRun("Verify that a session of a different SNI is not dumped.")
@@ -150,10 +152,12 @@
 tr = Test.AddTestRun("Verify the json content of the first session")
 verify_replay = "verify_replay.py"
 tr.Setup.CopyAs(verify_replay, Test.RunDirectory)
-tr.Processes.Default.Command = "python3 {0} {1} {2}".format(
+tr.Processes.Default.Command = 'python3 {0} {1} {2} --client-protocols "{3}" --client-tls-features "{4}"'.format(
         verify_replay,
         os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'),
-        replay_file_session_1)
+        replay_file_session_1,
+        session_1_protocols,
+        session_1_tls_features)
 tr.Processes.Default.ReturnCode = 0
 tr.StillRunningAfter = server
 tr.StillRunningAfter = ts

tests/gold_tests/pluginTest/traffic_dump/verify_replay.py

@@ -139,6 +139,40 @@ def verify_sensitive_fields_not_dumped(replay_json, sensitive_fields):
     return True
 
 
+def verify_client_protocols(replay_json, expected_protocol_features):
+    expected_protocols_list = expected_protocol_features.split(',')
+    expected_protocols_list.sort()
+    try:
+        protocol_node = replay_json['sessions'][0]['protocol']
+        protocol_list = protocol_node.copy()
+        protocol_list.sort()
+        if protocol_list == expected_protocols_list:
+            return True
+        else:
+            print('Unexpected protocol stack. Expected: "{}", found: "{}".'.format(
+                ','.join(expected_protocols_list), ','.join(protocol_list)))
+            return False
+    except KeyError:
+        print("Could not find client protocol stack node in the replay file.")
+        return False
+
+
+def verify_client_tls_features(replay_json, expected_tls_features):
+    try:
+        session = replay_json['sessions'][0]
+        for expected_tls_feature in expected_tls_features.split(','):
+            expected_key, expected_value = expected_tls_feature.split(':')
+            tls_features = session['tls']
+            try:
+                return tls_features[expected_key] == expected_value
+            except KeyError:
+                print("Could not find client tls feature in the replay file: {}".format(expected_key))
+                return False
+    except KeyError:
+        print("Could not find client tls node in the replay file.")
+        return False
+
+
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("schema_file",
@@ -155,6 +189,10 @@ def parse_args():
     parser.add_argument("--sensitive-fields",
                         action="append",
                         help="The fields that are considered sensitive and replaced with insensitive values.")
+    parser.add_argument("--client-protocols",
+                        help="The comma-separated protocol features to expect for the client connection.")
+    parser.add_argument("--client-tls-features",
+                        help="The TLS values to expect for the client connection.")
     return parser.parse_args()
 
 
@@ -188,6 +226,12 @@ def main():
     if args.sensitive_fields and not verify_sensitive_fields_not_dumped(replay_json, args.sensitive_fields):
         return 1
 
+    if args.client_protocols and not verify_client_protocols(replay_json, args.client_protocols):
+        return 1
+
+    if args.client_tls_features and not verify_client_tls_features(replay_json, args.client_tls_features):
+        return 1
+
     return 0