From 7016e9cc0e2d27d8bae5f9f8e2ca2d3a80b556d5 Mon Sep 17 00:00:00 2001 From: Wesley Rosenblum <55108558+WesleyRosenblum@users.noreply.github.com> Date: Thu, 24 Aug 2023 16:30:59 -0400 Subject: [PATCH] ci: require more interop tests to pass (#1919) * ci: require more interop tests to pass * add newline * fix whitespace * remove handshakeloss and corruption, fix handling of PR report * remove kwik * remove rebind address * patch rebind addr test * increase timeout limits and disable MTU probing to increase chance of HandshakeLoss/Corruption tests succeeding * remove msquic as it always fails everything and takes a long time --- .github/interop/check.py | 16 +- .github/interop/required.check.sh | 2 +- .github/interop/required.json | 460 ++++++++++++++++++++------ .github/interop/runner.patch | 27 +- quic/s2n-quic-qns/etc/run_endpoint.sh | 6 +- quic/s2n-quic-qns/src/limits.rs | 4 +- 6 files changed, 396 insertions(+), 119 deletions(-) diff --git a/.github/interop/check.py b/.github/interop/check.py index d77ff84a95..d325879a31 100644 --- a/.github/interop/check.py +++ b/.github/interop/check.py @@ -43,12 +43,20 @@ def format_required_report(report): outcome.setdefault(test['name'], {}) info = outcome[test['name']] - info.setdefault(client, {'client': False, 'server': False}) - info.setdefault(server, {'client': False, 'server': False}) + if client == s2n_quic and server == s2n_quic: + info.setdefault('s2n-quic', {}) + else: + info.setdefault(client, {}) + info.setdefault(server, {}) success = test['result'] == 'succeeded' - info[client]['client'] = success - info[server]['server'] = success + if client == s2n_quic and server == s2n_quic: + info['s2n-quic']['client'] = success + info['s2n-quic']['server'] = success + elif client == s2n_quic: + info[server]['server'] = success + else: + info[client]['client'] = success return outcome diff --git a/.github/interop/required.check.sh b/.github/interop/required.check.sh index 0eb2e014f8..b32d124721 100755 --- a/.github/interop/required.check.sh +++ b/.github/interop/required.check.sh @@ -32,7 +32,7 @@ for commit in $COMMITS; do fi if [ -s $OUT_DIR/$commit.json ]; then - python .github/interop/check.py --required .github/interop/required.json $OUT_DIR/$commit.json + python3 .github/interop/check.py --required .github/interop/required.json $OUT_DIR/$commit.json else echo " report could not be found; skipping" fi diff --git a/.github/interop/required.json b/.github/interop/required.json index 8327e81cc7..af14ac85c7 100644 --- a/.github/interop/required.json +++ b/.github/interop/required.json @@ -1,107 +1,184 @@ { "ipv6": { "aioquic": [ - "client" + "client", + "server" ], "kwik": [], "lsquic": [ - "client" + "client", + "server" + ], + "mvfst": [ + "client", + "server" ], - "msquic": [], - "mvfst": [], "neqo": [ - "client" + "client", + "server" ], "ngtcp2": [ - "client" + "client", + "server" ], "picoquic": [ - "client" + "client", + "server" ], "quic-go": [ - "client" + "client", + "server" ], "quiche": [ - "client" + "client", + "server" ], - "xquic": [] + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" + ], + "xquic": [ + "server" + ] }, "handshake": { "aioquic": [ - "client" + "client", + "server" ], "kwik": [], "lsquic": [ - "client" + "client", + "server" + ], + "mvfst": [ + "client", + "server" ], - "msquic": [], - "mvfst": [], "neqo": [ - "client" + "client", + "server" ], "ngtcp2": [ - "client" + "client", + "server" + ], + "picoquic": [ + "client", + "server" ], - "picoquic": [], "quic-go": [ - "client" + "client", + "server" ], "quiche": [ - "client" + "client", + "server" ], - "xquic": [] + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" + ], + "xquic": [ + "server" + ] }, "transfer": { "aioquic": [ - "client" + "client", + "server" ], "kwik": [], "lsquic": [ + "client", + "server" + ], + "mvfst": [ "client" ], - "msquic": [], - "mvfst": [], "neqo": [ - "client" + "client", + "server" ], "ngtcp2": [ - "client" + "client", + "server" ], "picoquic": [ - "client" + "client", + "server" ], "quic-go": [ - "client" + "client", + "server" ], "quiche": [ - "client" + "client", + "server" + ], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" ], "xquic": [] }, "longrtt": { "aioquic": [ - "client" + "client", + "server" ], "kwik": [], - "lsquic": [], - "msquic": [], - "mvfst": [], + "lsquic": [ + "client", + "server" + ], + "mvfst": [ + "client", + "server" + ], "neqo": [ - "client" + "client", + "server" ], "ngtcp2": [ - "client" + "client", + "server" ], "picoquic": [ - "client" + "client", + "server" ], "quic-go": [ - "client" + "client", + "server" ], "quiche": [ - "client" + "client", + "server" ], - "xquic": [] + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" + ], + "xquic": [ + "server" + ] }, "chacha20": { "aioquic": [ @@ -109,7 +186,6 @@ ], "kwik": [], "lsquic": [], - "msquic": [], "mvfst": [], "neqo": [ "client" @@ -120,84 +196,136 @@ "picoquic": [ "client" ], - "quic-go": [], + "quic-go": [ + "client" + ], "quiche": [], + "s2n-quic": [], "xquic": [] }, "multiplexing": { "aioquic": [ - "client" + "client", + "server" ], "kwik": [], - "lsquic": [], - "msquic": [], - "mvfst": [], - "neqo": [], + "lsquic": [ + "client", + "server" + ], + "mvfst": [ + "client", + "server" + ], + "neqo": [ + "client", + "server" + ], "ngtcp2": [ - "client" + "client", + "server" ], "picoquic": [ - "client" + "client", + "server" ], "quic-go": [ - "client" + "client", + "server" ], "quiche": [ - "client" + "client", + "server" + ], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" ], "xquic": [] }, "retry": { "aioquic": [ - "client" + "client", + "server" ], "kwik": [], "lsquic": [ - "client" + "client", + "server" ], - "msquic": [], "mvfst": [], "neqo": [ - "client" + "client", + "server" ], "ngtcp2": [ - "client" + "client", + "server" ], "picoquic": [ - "client" + "client", + "server" ], "quic-go": [ - "client" + "client", + "server" ], "quiche": [ - "client" + "client", + "server" + ], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" ], "xquic": [] }, "blackhole": { "aioquic": [ - "client" + "client", + "server" ], "kwik": [], "lsquic": [ - "client" + "client", + "server" ], - "msquic": [], "mvfst": [], "neqo": [ - "client" + "client", + "server" ], "ngtcp2": [ - "client" + "client", + "server" ], "picoquic": [ - "client" + "client", + "server" ], "quic-go": [ - "client" + "client", + "server" ], "quiche": [ - "client" + "client", + "server" + ], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" ], "xquic": [] }, @@ -205,7 +333,6 @@ "aioquic": [], "kwik": [], "lsquic": [], - "msquic": [], "mvfst": [], "neqo": [ "client" @@ -220,6 +347,7 @@ "client" ], "quiche": [], + "s2n-quic": [], "xquic": [] }, "ecn": { @@ -228,73 +356,129 @@ "lsquic": [ "client" ], - "msquic": [], "mvfst": [], "neqo": [], "ngtcp2": [ - "client" + "client", + "server" ], "picoquic": [ "client" ], "quic-go": [], "quiche": [], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" + ], "xquic": [] }, "amplificationlimit": { - "aioquic": [], + "aioquic": [ + "client", + "server" + ], "kwik": [], - "lsquic": [], - "msquic": [], + "lsquic": [ + "client", + "server" + ], "mvfst": [], "neqo": [ "client" ], - "ngtcp2": [], + "ngtcp2": [ + "client", + "server" + ], "picoquic": [ - "client" + "client", + "server" ], "quic-go": [ - "client" + "client", + "server" ], - "quiche": [], - "xquic": [] + "quiche": [ + "client", + "server" + ], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" + ], + "xquic": [ + "server" + ] }, "handshakeloss": { "aioquic": [], "kwik": [], "lsquic": [], - "msquic": [], "mvfst": [], "neqo": [], "ngtcp2": [], "picoquic": [], "quic-go": [], "quiche": [], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" + ], "xquic": [] }, "transferloss": { "aioquic": [ - "client" + "client", + "server" ], "kwik": [], "lsquic": [ + "client", + "server" + ], + "mvfst": [ "client" ], - "msquic": [], - "mvfst": [], "neqo": [ - "client" + "client", + "server" ], "ngtcp2": [ - "client" + "client", + "server" + ], + "picoquic": [ + "client", + "server" ], - "picoquic": [], "quic-go": [ - "client" + "client", + "server" ], "quiche": [ - "client" + "client", + "server" + ], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" ], "xquic": [] }, @@ -302,79 +486,139 @@ "aioquic": [], "kwik": [], "lsquic": [], - "msquic": [], "mvfst": [], "neqo": [], "ngtcp2": [], "picoquic": [], "quic-go": [], "quiche": [], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" + ], "xquic": [] }, "transfercorruption": { "aioquic": [ - "client" + "client", + "server" ], "kwik": [], "lsquic": [ + "client", + "server" + ], + "mvfst": [ "client" ], - "msquic": [], - "mvfst": [], "neqo": [ - "client" + "client", + "server" ], "ngtcp2": [ - "client" + "client", + "server" ], "picoquic": [ - "client" + "client", + "server" ], "quic-go": [ - "client" + "client", + "server" ], "quiche": [ - "client" + "client", + "server" + ], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" ], "xquic": [] }, "rebind-addr": { - "aioquic": [], + "aioquic": [ + "server" + ], "kwik": [], - "lsquic": [], - "msquic": [], + "lsquic": [ + "client" + ], "mvfst": [], - "neqo": [], + "neqo": [ + "client" + ], "ngtcp2": [], - "picoquic": [], - "quic-go": [], - "quiche": [], + "picoquic": [ + "client" + ], + "quic-go": [ + "client" + ], + "quiche": [ + "client" + ], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" + ], "xquic": [] }, "rebind-port": { "aioquic": [], "kwik": [], - "lsquic": [], - "msquic": [], + "lsquic": [ + "client" + ], "mvfst": [], "neqo": [], - "ngtcp2": [], - "picoquic": [], - "quic-go": [], - "quiche": [], + "ngtcp2": [ + "client" + ], + "picoquic": [ + "client" + ], + "quic-go": [ + "client" + ], + "quiche": [ + "client" + ], + "s2n-quic": [ + "client", + "server" + ], + "s2n-quic-rustls": [ + "client", + "server" + ], "xquic": [] }, "connectionmigration": { "aioquic": [], "kwik": [], "lsquic": [], - "msquic": [], "mvfst": [], "neqo": [], "ngtcp2": [], "picoquic": [], "quic-go": [], "quiche": [], + "s2n-quic": [], + "s2n-quic-rustls": [], "xquic": [] } } diff --git a/.github/interop/runner.patch b/.github/interop/runner.patch index e1e695b4af..ad56b1a11f 100644 --- a/.github/interop/runner.patch +++ b/.github/interop/runner.patch @@ -49,7 +49,7 @@ index 7541cae..ba1b4da 100644 - VERSION=$VERSION depends_on: diff --git a/implementations.json b/implementations.json -index 9150551..55aaa4a 100644 +index 9150551..fc21610 100644 --- a/implementations.json +++ b/implementations.json @@ -9,11 +9,6 @@ @@ -64,7 +64,19 @@ index 9150551..55aaa4a 100644 "mvfst": { "image": "lnicco/mvfst-qns:latest", "url": "https://github.com/facebookincubator/mvfst", -@@ -79,8 +74,13 @@ +@@ -49,11 +44,6 @@ + "url": "https://quic.nginx.org/", + "role": "server" + }, +- "msquic": { +- "image": "ghcr.io/microsoft/msquic/qns:main", +- "url": "https://github.com/microsoft/msquic", +- "role": "both" +- }, + "chrome": { + "image": "martenseemann/chrome-quic-interop-runner", + "url": "https://github.com/marten-seemann/chrome-quic-interop-runner", +@@ -79,8 +69,13 @@ "url": "https://github.com/quinn-rs/quinn", "role": "both" }, @@ -195,7 +207,7 @@ index fbd9515..aa8d6ed 100755 - sys.exit(main()) + main() diff --git a/testcases.py b/testcases.py -index 6d7ecfb..f2f9dea 100644 +index 6d7ecfb..d7c323c 100644 --- a/testcases.py +++ b/testcases.py @@ -90,6 +90,10 @@ class TestCase(abc.ABC): @@ -209,6 +221,15 @@ index 6d7ecfb..f2f9dea 100644 @staticmethod def scenario() -> str: """ Scenario for the ns3 simulator """ +@@ -1181,7 +1185,7 @@ class TestCasePortRebinding(TestCaseTransfer): + @staticmethod + def scenario() -> str: + """ Scenario for the ns3 simulator """ +- return "rebind --delay=15ms --bandwidth=10Mbps --queue=25 --first-rebind=1s --rebind-freq=5s" ++ return "rebind --delay=15ms --bandwidth=10Mbps --queue=25 --first-rebind=2s --rebind-freq=5s" + + def check(self) -> TestResult: + if not self._keylog_file(): @@ -1203,54 +1207,26 @@ class TestCasePortRebinding(TestCaseTransfer): logging.info("Server saw only a single client port in use; test broken?") return TestResult.FAILED diff --git a/quic/s2n-quic-qns/etc/run_endpoint.sh b/quic/s2n-quic-qns/etc/run_endpoint.sh index db91e799da..ec61b880c1 100755 --- a/quic/s2n-quic-qns/etc/run_endpoint.sh +++ b/quic/s2n-quic-qns/etc/run_endpoint.sh @@ -51,13 +51,17 @@ if [ "$QNS_MODE" == "interop" ]; then CLIENT_PARAMS+=" --application-protocols h3" fi - if [ "$TESTCASE" == "ecn" ]; then + if [ "$TESTCASE" == "ecn" ] || [ "$TESTCASE" == "multiconnect" ]; then # The ECN test requires all packets set an ECT codepoint to pass. s2n-quic temporarily # stops setting the ECT codepoint after 10 packets in case faulty equipment drops # ECT marked packets (see https://datatracker.ietf.org/doc/html/rfc9000#section-13.4.2) # To increase the chance this test completes in 10 packets or less, we disable # MTU probing to avoid sending MTU probes that contribute to the 10 packet limit. # 1228 is the minimum allowed MaxMtu, see `s2n_quic_core::path::MaxMtu::MIN` + + # multiconnect refers to the HandshakeLoss and HandshakeCorruption tests. Due to the + # severe loss/corrupt rates these tests use (30%), we also disable MTU probing for + # them to not waste any packets on MTU probes, increasing the chance for success. SERVER_PARAMS+=" --max-mtu 1228" CLIENT_PARAMS+=" --max-mtu 1228" fi diff --git a/quic/s2n-quic-qns/src/limits.rs b/quic/s2n-quic-qns/src/limits.rs index f8eb956cbd..762a14d8ea 100644 --- a/quic/s2n-quic-qns/src/limits.rs +++ b/quic/s2n-quic-qns/src/limits.rs @@ -19,10 +19,10 @@ pub struct Limits { impl Limits { // Increase the MaxHandshakeDuration from the default of 10 seconds - const MAX_HANDSHAKE_DURATION: Duration = Duration::from_secs(60); + const MAX_HANDSHAKE_DURATION: Duration = Duration::from_secs(300); // Increase MaxIdleTimeout from the default of 30 seconds - const MAX_IDLE_TIMEOUT: Duration = Duration::from_secs(60); + const MAX_IDLE_TIMEOUT: Duration = Duration::from_secs(300); pub fn limits(&self) -> s2n_quic::provider::limits::Limits { let data_window = self.data_window();