Moving away from Scapy-ssl_tls and fix tls/ tests

README.md

@@ -71,8 +71,10 @@ deproxy server, and workload tests should use wrk client and nginx server.
 ## Requirements
 
 - Host for testing framework: `Python2`, `python2-paramiko`,
-`python-configparser`, `python-subprocess32`, `wrk`, `ab`, `python-scapy`,
-`python-cryptography`, `scapy-ssl_tls` (installed with `pip`), `h2spec`
+`python-configparser`, `python-subprocess32`, `wrk`, `ab`, `scapy`
+(Debian 10 contains outdated 2.4.0 and we require 2.4.4, so install it with pip),
+`python-cryptography`, `h2spec`, `pycryptodomex`, and `tinyec` (the last
+two can be installed using `pip install pycryptodomex tinyec`)
 - All hosts except previous one: `sftp-server`
 - Host for running TempestaFW: Linux kernel with Tempesta, TempestaFW sources,
 `systemtap`, `tcpdump`, `bc`

helpers/analyzer.py

@@ -36,7 +36,9 @@ def __init__(self, node, host, count=0,
         self.packets = []
         self.dump_file = '/tmp/tmp_packet_dump'
         str_ports = ' or '.join(('tcp port %s' % p) for p in ports)
-        cmd = 'timeout %s tcpdump -i any %s -w - %s || true'
+        # TODO #120: it's bad to use timeout(1). Instead we should run
+        # the tcpdump process and kill it when the test is done.
+        cmd = 'timeout %s tcpdump -i any -n %s -w - %s || true'
         count_flag = ('-c %s' % count) if count else ''
         self.cmd = cmd % (timeout, count_flag, str_ports)
         self.err_msg = ' '.join(["Can't %s sniffer on", host])
@@ -160,18 +162,22 @@ def check_results(self):
                 self.srv_pkts += [int(plen)]
         (tfw_n, srv_n) = (len(self.tfw_pkts), len(self.srv_pkts))
         assert tfw_n and srv_n, "Traffic wasn't captured"
-        if tfw_n > srv_n:
+        assert tfw_n > 3, "Captured the number of packets less than" \
+                          " the TCP/TLS overhead"
+        if tfw_n > srv_n + 2:
             tf_cfg.dbg(4, "Tempesta TLS generates more packets (%d) than" \
-                          " original server (%d)" % (tfw_n, srv_n))
+                          " original server (%d) plus the TLS overhead (2 segs)"
+                          % (tfw_n, srv_n))
             res = False
         # We're good if Tempesta generates less number of packets than
-        # the server.
-        for i in xrange(tfw_n):
-            if i >= srv_n:
+        # the server. We skip the initial TCP SYN-ACK, and the TLS 1.2
+        # 2-RTT handshake overhead. This may fail for TLS 1.3.
+        for i in xrange(3, tfw_n):
+            if i - 2 >= srv_n:
                 tf_cfg.dbg(4, "Extra packet %d, size=%d"
                               % (i, self.tfw_pkts[i]))
                 res = False
-            elif self.tfw_pkts[i] < self.srv_pkts[i]:
+            elif self.tfw_pkts[i] < self.srv_pkts[i - 2]:
                 tf_cfg.dbg(4, "Tempesta packet %d less than server's" \
                                " (%d < %d)"
                                % (i, self.tfw_pkts[i], self.srv_pkts[i]))

helpers/remote.py

@@ -78,6 +78,9 @@ def run_cmd(self, cmd, timeout=DEFAULT_TIMEOUT, ignore_stderr=False,
         with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
                               stderr=stderr_pipe, env=env_full) as p:
             try:
+                # TODO #120: we should provide kill() and pid() interfaces to
+                # let caller to determine if the command is executed and
+                # when it's terminated and/or teriminate if when necessary.
                 stdout, stderr = p.communicate(timeout)
                 assert p.returncode == 0, \
                     "Cmd: '%s' return code is not 0 (%d)." % (cmd, p.returncode)
@@ -162,6 +165,9 @@ def run_cmd(self, cmd, timeout=DEFAULT_TIMEOUT, ignore_stderr=False,
             ])
             tf_cfg.dbg(4, "\tEffective command '%s' after injecting environment" % cmd)
         try:
+            # TODO #120: the same as for LocalNode - provide an interface to check
+            # whether the command is executed and when it's terminated and/or
+            # kill it when necessary.
             _, out_f, err_f = self.ssh.exec_command(cmd, timeout=timeout)
             stdout = out_f.read()
             if not ignore_stderr:

tests_disabled.json

@@ -81,10 +81,6 @@
             "name": "malformed.test_malformed_headers",
             "reason" : "See tempesta-tech/tempesta#1050 and tempesta-tech/tempesta#1053"
         },
-        {
-            "name" : "tls.test_tls_integrity.Proxy.test_tcp_segs",
-            "reason" : "#1325: the test fails before the issue is fixed. Probably there is a reason for the behavior and the test should be reworked."
-        },
         {
             "name" : "sessions.test_redir_mark.RedirectMark.test_rmark_wo_or_incorrect_cookie",
             "reason" : "#861: TCP: reset blocked connections"

tls/__init__.py

@@ -1,4 +1,5 @@
 __all__ = ['test_tls_kernel', 'test_tls_stress', 'test_tls_handshake',
-           'test_tls_integrity', 'test_tls_cert']
+           'test_tls_integrity', 'test_tls_cert', 'test_tls_tickets',
+           'test_tls_limits']
 
 # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

tls/handshake.py

@@ -4,29 +4,30 @@
 implementation. This tool emphasises flexibility in generation of TLS traffic,
 not performance.
 
-ScaPy is still not fully compatible with Python3, but I still use __future__
-module for easier migration to Python3.
-https://github.com/tintinweb/scapy-ssl_tls/issues/39
+We use __future__ module for easier migration to Python3.
 
 TLS 1.2 is specified in RFC 5246. See also these useful references:
  - https://wiki.osdev.org/SSL/TLS
  - https://wiki.osdev.org/TLS_Handshake
  - https://wiki.osdev.org/TLS_Encryption
+
+Debugging can be simplified by enabling verbose in TlsHandshake instance. Or
+by running `tshark -i lo -f 'port 443' -Y ssl -O ssl`
 """
 from __future__ import print_function
 from contextlib import contextmanager
 import random
 import socket
 import ssl # OpenSSL based API
 import struct
-import scapy_ssl_tls.ssl_tls as tls
 from time import sleep
 
 from helpers import dmesg, tf_cfg
 from helpers.error import Error
+from scapy_ssl_tls import ssl_tls as tls
 
 __author__ = 'Tempesta Technologies, Inc.'
-__copyright__ = 'Copyright (C) 2018-2019 Tempesta Technologies, Inc.'
+__copyright__ = 'Copyright (C) 2018-2020 Tempesta Technologies, Inc.'
 __license__ = 'GPL2'
 
 
@@ -42,19 +43,19 @@ def x509_check_cn(cert, cn):
     generated by the cryptography library, so we can not use full string
     matching and have to use substring matching instead.
     """
-    for f in cert.fields['subject']:
-        if f.oid.val == '2.5.4.3':
-            return str(f.value).endswith(cn)
+    for f in cert.tbsCertificate.issuer:
+        if f.rdn[0].type.val == '2.5.4.3':
+            return str(f.rdn[0].value).endswith(cn)
     raise Error("Certificate has no CommonName")
 
 
 def x509_check_issuer(cert, issuer):
     """
     The same as above, but for Issuer OrganizationName (O, OID '2.5.4.10').
     """
-    for f in cert.fields['issuer']:
-        if f.oid.val == '2.5.4.10':
-            return str(f.value).endswith(issuer)
+    for f in cert.tbsCertificate.issuer:
+        if f.rdn[0].type.val == '2.5.4.10':
+            return str(f.rdn[0].value).endswith(issuer)
     raise Error("Certificate has no Issuer OrganizationName")
 
 
@@ -98,6 +99,26 @@ def __init__(self, addr=None, port=443, io_to=0.5, chunk=None,
         self.host = None
         # Server certificate.
         self.cert = None
+        # Random session ticket by default.
+        self.set_ticket_data('ticket_data')
+        # Session id, must be filled for resume
+        self.session_id = ''
+
+    def set_ticket_data(self, data):
+        """ Set session ticket data. Following values are possible:
+        - 'None' - session ticket extension will be disabled;
+        - '' (empty string) - empty session ticket extension will be added;
+        - '<str>' - arbitrary string will be inserted as session ticket.
+        'data' can be represented either as string or as
+        scapy_ssl_tls.TLSSessionTicket.
+        """
+        if data is None:
+            self.ticket_data = None
+            return
+        if type(data) is tls.TLSSessionTicket:
+            self.ticket_data = data.ticket
+        else:
+            self.ticket_data = data
 
     @contextmanager
     def socket_ctx(self):
@@ -212,7 +233,7 @@ def extra_extensions(self):
         # We're must be good with standard, but unsupported options.
         self.exts += [
             tls.TLSExtension(type=0x3), # TrustedCA, RFC 6066 6.
-            tls.TLSExtension(type=0x5), # StatusRequest, RFC 6066 8.
+            tls.TLSExtension() / tls.TLSExtCertificateStatusRequest(),
             tls.TLSExtension(type=0xf0), # Bad extension, just skipped
 
             tls.TLSExtension() /
@@ -221,7 +242,7 @@ def extra_extensions(self):
                 tls.TLSALPNProtocol(data="http/2.0")]),
 
             tls.TLSExtension() /
-            tls.TLSExtMaxFragmentLength(fragment_length=0x01), # 512 bytes
+            tls.TLSExtMaxFragmentLength(fragment_length=0x04), # 4096 bytes
 
             tls.TLSExtension() /
             tls.TLSExtCertificateURL(certificate_urls=[
@@ -230,10 +251,12 @@ def extra_extensions(self):
             tls.TLSExtension() /
             tls.TLSExtHeartbeat(
                 mode=tls.TLSHeartbeatMode.PEER_NOT_ALLOWED_TO_SEND),
-
-            tls.TLSExtension() /
-            tls.TLSExtSessionTicketTLS(data="myticket")
         ]
+        if self.ticket_data is not None:
+            self.exts += [
+                tls.TLSExtension() /
+                tls.TLSExtSessionTicketTLS(data=self.ticket_data)
+            ]
         return self.exts
 
     def send_12_alert(self, level, desc):
@@ -256,6 +279,8 @@ def _do_12_hs(self, fuzzer=None):
         c_h = tls.TLSClientHello(
             gmt_unix_time=0x22222222,
             random_bytes='\x11' * 28,
+            session_id=self.session_id,
+            session_id_length=len(self.session_id),
             cipher_suites=[
                 tls.TLSCipherSuite.ECDHE_ECDSA_WITH_AES_128_GCM_SHA256] +
                 self.ciphers,
@@ -279,7 +304,7 @@ def _do_12_hs(self, fuzzer=None):
         if not resp.haslayer(tls.TLSCertificate):
             return False
         self.cert = resp[tls.TLSCertificate].data
-        assert self.cert, "No cerfificate received"
+        assert self.cert, "No certificate received"
         if self.verbose:
             resp.show()
 
@@ -315,6 +340,66 @@ def _do_12_hs(self, fuzzer=None):
             print(self.sock.tls_ctx)
         return True
 
+    def _do_12_hs_resume(self, master_secret, ticket, fuzzer=None):
+        """
+        Test abbreviated TLS 1.2 handshake: establish a new TCP connection
+        and send predefined TLS handshake records.
+        """
+        try:
+            self.conn_estab()
+        except socket.error:
+            return False
+
+        self.sock.tls_ctx.resume_session(master_secret)
+        self.set_ticket_data(ticket)
+        # Session must be non-null for resumption.
+        self.session_id = '\x38' * 32
+
+        c_h = tls.TLSClientHello(
+            gmt_unix_time=0x22222222,
+            random_bytes='\x11' * 28,
+            session_id=self.session_id,
+            session_id_length=len(self.session_id),
+            cipher_suites=[
+                tls.TLSCipherSuite.ECDHE_ECDSA_WITH_AES_128_GCM_SHA256] +
+                self.ciphers,
+                extensions=[
+                    tls.TLSExtension(type=0x16), # Encrypt-then-MAC
+                    tls.TLSExtension() / tls.TLSExtECPointsFormat()]
+                + self.extra_extensions()
+        )
+        msg = tls.TLSRecord(version='TLS_1_2') / \
+            tls.TLSHandshakes(
+                handshakes=[tls.TLSHandshake() / c_h]
+            )
+        if self.verbose:
+            msg.show()
+
+        # Send ClientHello and read ServerHello, ServerCertificate,
+        # ServerKeyExchange, ServerHelloDone.
+        self.inject_bad(fuzzer)
+        resp = self.send_recv(msg)
+        if not resp.haslayer(tls.TLSChangeCipherSpec):
+            return False
+        if self.verbose:
+            resp.show()
+
+        msg = tls.TLSRecord(version='TLS_1_2') / tls.TLSChangeCipherSpec()
+        if self.verbose:
+            msg.show()
+        self.inject_bad(fuzzer)
+        self.sock.sendall(tls.TLS.from_records([msg]))
+        # Now we can calculate the final session checksum, send ClientFinished.
+        cf_h = tls.TLSHandshakes(
+            handshakes=[tls.TLSHandshake() /
+                        tls.TLSFinished(
+                            data=self.sock.tls_ctx.get_verify_data())])
+        msg = tls.TLSRecord(version='TLS_1_2') / cf_h
+        if self.verbose:
+            msg.show()
+        self.send_recv(tls.TLS.from_records([msg]))
+        return True
+
     def __get_host(self):
         if self.host:
             return self.host
@@ -347,6 +432,11 @@ def do_12(self, fuzzer=None):
                 return False
             return self._do_12_req(fuzzer)
 
+    def do_12_resume(self, master_secret, ticket, fuzzer=None):
+        with self.socket_ctx():
+            if not self._do_12_hs_resume(master_secret, ticket, fuzzer):
+                return False
+            return self._do_12_req(fuzzer)
 
 class TlsHandshakeStandard:
     """
@@ -369,15 +459,15 @@ def try_tls_vers(self, version):
         sock.connect((self.addr, self.port))
         try:
             tls_sock = ssl.wrap_socket(sock, ssl_version=version)
-        except IOError:
-            # Exception on client side TLS with established TCP connection -
-            # we're good with connection rejection, this means that Tempesta
-            # correctly doesn't establish bad TLS handshake.
-            sock.close()
-            if klog.warn_count(TLS_HS_WARN) == 1:
+        except ssl.SSLError as e:
+            # Correct connection termination with PROTOCOL_VERSION alert.
+            if e.reason == "TLSV1_ALERT_PROTOCOL_VERSION":
                 return True
+        except IOError as e:
             if self.verbose:
                 print("TLS handshake failed w/o warning")
+        if self.verbose:
+            print("Connection of unsupported TLS 1.%d established" % version)
         return False
 
     def do_old(self):

tls/scapy_ssl_tls/__init__.py

@@ -0,0 +1,4 @@
+__all__ = ['pkcs7', 'ssl_tls_automata', 'ssl_tls_crypto', 'ssl_tls_keystore',
+           'ssl_tls', 'ssl_tls_registry']
+
+# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4