OpenSSL Support (Linux) (#11)

This PR updates tls_openssl.c to the newest OpenSSL APIs for QUIC. It also gets OpenSSL building with cmake.

Also contains a push of the latest internal code changes.

.gitmodules

@@ -1,3 +1,7 @@
 [submodule "submodules/googletest"]
 	path = submodules/googletest
 	url = https://github.com/google/googletest
+[submodule "submodules/openssl"]
+	path = submodules/openssl
+	url = https://github.com/tatsuhiro-t/openssl.git
+	branch = openssl-quic-draft-24

BUILD.md

@@ -15,6 +15,14 @@ to get all the submodules.
 - Run `cmake -G "Visual Studio 16 2019" -A x64 ..`
 - Run `cmake --build . --config RELEASE`
 
+### Building with OpenSSL
+
+**Requirements**
+  * [Perl](https://www.perl.org/)
+  * [NMAKE](https://docs.microsoft.com/en-us/cpp/build/reference/nmake-reference?view=vs-2019)
+
+**TODO** - Figure out the correct set of steps.
+
 ### Running the tests
 
 There is a one time registry setup required before the tests can be run when using

CMakeLists.txt

@@ -3,26 +3,26 @@
 
 cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
 
-message("-- Source Dir: ${CMAKE_SOURCE_DIR}")
-message("-- Host System name: ${CMAKE_HOST_SYSTEM_NAME}")
+message(STATUS "Source Dir: ${CMAKE_SOURCE_DIR}")
+message(STATUS "Host System name: ${CMAKE_HOST_SYSTEM_NAME}")
 if ("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Windows")
     set(CMAKE_SYSTEM_VERSION 10.0.18362.0 CACHE STRING INTERNAL FORCE)
     set(CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION 10.0.18362.0 CACHE STRING INTERNAL FORCE)
 endif()
 
 project(msquic)
 
-message("-- System name: ${CMAKE_SYSTEM_NAME}")
-message("-- System version: ${CMAKE_SYSTEM_VERSION}")
-message("-- Platform version: ${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}")
+message(STATUS "System name: ${CMAKE_SYSTEM_NAME}")
+message(STATUS "System version: ${CMAKE_SYSTEM_VERSION}")
+message(STATUS "Platform version: ${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}")
 
 enable_testing()
 
 # Set the default TLS method for each platform.
 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
     set(QUIC_TLS "schannel" CACHE STRING "TLS Library to use")
 else()
-    set(QUIC_TLS "stub" CACHE STRING "TLS Library to use")
+    set(QUIC_TLS "openssl" CACHE STRING "TLS Library to use")
 endif()
 
 option(QUIC_BUILD_TOOLS "Builds the tools code" ON)
@@ -55,6 +55,22 @@ else()
     set(QUIC_CXX_FLAGS "-DQUIC_PLATFORM_LINUX -DQUIC_EVENTS_SYSLOG -DQUIC_LOGS_SYSLOG -fms-extensions -fPIC -Wall -Wno-reorder -Wno-unknown-pragmas -Wno-unused-variable -Wno-unused-value -Wno-sign-compare -Wno-format --std=c++0x -g -pthread")
 endif()
 
+if(QUIC_TLS STREQUAL "openssl")
+    # Configure and build OpenSSL.
+    add_custom_command(
+        WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/submodules/openssl
+        OUTPUT ${CMAKE_SOURCE_DIR}/bld/openssl/include
+        OUTPUT ${CMAKE_SOURCE_DIR}/bld/openssl/lib/libcrypto.so
+        OUTPUT ${CMAKE_SOURCE_DIR}/bld/openssl/lib/libssl.so
+        COMMAND ./Configure linux-x86_64 enable-tls1_3 --prefix=${CMAKE_SOURCE_DIR}/bld/openssl
+        COMMAND make -j$(nproc)
+        COMMAND make install_sw)
+    add_custom_target(OpenSSL
+        DEPENDS ${CMAKE_SOURCE_DIR}/bld/openssl/include
+        DEPENDS ${CMAKE_SOURCE_DIR}/bld/openssl/lib/libcrypto.so
+        DEPENDS ${CMAKE_SOURCE_DIR}/bld/openssl/lib/libssl.so)
+endif()
+
 include_directories(${CMAKE_SOURCE_DIR}/inc)
 
 # Product code

azure-pipelines.yml

@@ -11,6 +11,11 @@ strategy:
       imageName: 'ubuntu-latest'
       cmakeArgs: '-g ''Linux Makefiles'' -DQUIC_TLS=stub'
       testCmd: 'bash ./test/run_linux.sh'
+    linux-openssl:
+      platform: 'Linux-OpenSSL'
+      imageName: 'ubuntu-latest'
+      cmakeArgs: '-g ''Linux Makefiles'' -DQUIC_TLS=openssl'
+      testCmd: 'bash ./test/run_linux.sh'
     windows-x64-stub:
       platform: 'Windows-x64-Stub'
       imageName: 'windows-latest'

core/binding.c

@@ -1109,7 +1109,7 @@ QuicBindingCreateConnection(
     }
 
     BindingRefAdded = TRUE;
-    NewConnection->Binding = Binding;
+    NewConnection->Paths[0].Binding = Binding;
     InterlockedIncrement(&Binding->HandshakeConnections);
     InterlockedExchangeAdd64(
         (LONG64*)&MsQuicLib.CurrentHandshakeMemoryUsage,

core/cid.h

@@ -97,6 +97,11 @@ typedef struct _QUIC_CID {
     //
     uint8_t Acknowledged : 1;
     //
+    // Used for destination CIDs. The CID has been locally assigned to a path
+    // and can't be used for any other path.
+    //
+    uint8_t UsedLocally : 1;
+    //
     // Used for source CIDs. The peer has sent a packet that used this CID.
     //
     uint8_t UsedByPeer : 1;

core/congestion_control.c

@@ -96,7 +96,7 @@ QuicCongestionControlInitialize(
     Cc->SlowStartThreshold = UINT32_MAX;
     Cc->SendIdleTimeoutMs = Settings->SendIdleTimeoutMs;
     Cc->InitialWindowPackets = Settings->InitialWindowPackets;
-    Cc->CongestionWindow = Connection->Send.PathMtu * Cc->InitialWindowPackets;
+    Cc->CongestionWindow = Connection->Paths[0].Mtu * Cc->InitialWindowPackets;
     Cc->BytesInFlightMax = Cc->CongestionWindow / 2;
     QuicConnLogOutFlowStats(Connection);
     QuicConnLogCubic(Connection);
@@ -112,7 +112,7 @@ QuicCongestionControlReset(
     Cc->SlowStartThreshold = UINT32_MAX;
     Cc->IsInRecovery = FALSE;
     Cc->HasHadCongestionEvent = FALSE;
-    Cc->CongestionWindow = Connection->Send.PathMtu * Cc->InitialWindowPackets;
+    Cc->CongestionWindow = Connection->Paths[0].Mtu * Cc->InitialWindowPackets;
     Cc->BytesInFlightMax = Cc->CongestionWindow / 2;
     Cc->BytesInFlight = 0;
     QuicConnLogOutFlowStats(Connection);
@@ -138,7 +138,9 @@ QuicCongestionControlPredictNextWindow(
             Wnd = Cc->SlowStartThreshold;
         }
     } else {
-        Wnd = Cc->CongestionWindow + QuicCongestionControlGetConnection(Cc)->Send.PathMtu;
+        Wnd =
+            Cc->CongestionWindow +
+            QuicCongestionControlGetConnection(Cc)->Paths[0].Mtu;
     }
     return Wnd;
 }
@@ -159,7 +161,7 @@ QuicCongestionControlGetSendAllowance(
         //
         SendAllowance = 0;
 
-    } else if (!Connection->State.UsePacing || !Connection->State.GotFirstRttSample) {
+    } else if (!Connection->State.UsePacing || !Connection->Paths[0].GotFirstRttSample) {
         //
         // Pacing is disabled or we don't have an RTT sample yet, so just send
         // everything we can.
@@ -172,8 +174,8 @@ QuicCongestionControlGetSendAllowance(
         // be split into chunks which are spread out over the RTT.
         // SendAllowance will be set to the size of the next chunk.
         //
-        uint32_t MinChunkSize = QUIC_SEND_PACING_MIN_CHUNK * Connection->Send.PathMtu;
-        if (Connection->SmoothedRtt < MS_TO_US(QUIC_SEND_PACING_INTERVAL) ||
+        uint32_t MinChunkSize = QUIC_SEND_PACING_MIN_CHUNK * Connection->Paths[0].Mtu;
+        if (Connection->Paths[0].SmoothedRtt < MS_TO_US(QUIC_SEND_PACING_INTERVAL) ||
             Cc->CongestionWindow < MinChunkSize ||
             !TimeSinceLastSendValid) {
             //
@@ -202,7 +204,7 @@ QuicCongestionControlGetSendAllowance(
             uint64_t EstimatedWnd = QuicCongestionControlPredictNextWindow(Cc);
 
             SendAllowance =
-                (uint32_t)((EstimatedWnd * TimeSinceLastSend) / Connection->SmoothedRtt);
+                (uint32_t)((EstimatedWnd * TimeSinceLastSend) / Connection->Paths[0].SmoothedRtt);
             if (SendAllowance < MinChunkSize) {
                 SendAllowance = MinChunkSize;
             }
@@ -276,15 +278,15 @@ QuicCongestionControlOnCongestionEvent(
     //
     Cc->KCubic =
         CubeRoot(
-            (Cc->WindowMax / Connection->Send.PathMtu * (10 - TEN_TIMES_BETA_CUBIC) << 9) /
+            (Cc->WindowMax / Connection->Paths[0].Mtu * (10 - TEN_TIMES_BETA_CUBIC) << 9) /
             TEN_TIMES_C_CUBIC);
     Cc->KCubic = S_TO_MS(Cc->KCubic);
     Cc->KCubic >>= 3;
 
     Cc->SlowStartThreshold =
     Cc->CongestionWindow =
         max(
-            (uint32_t)Connection->Send.PathMtu * Cc->InitialWindowPackets,
+            (uint32_t)Connection->Paths[0].Mtu * Cc->InitialWindowPackets,
             Cc->CongestionWindow * TEN_TIMES_BETA_CUBIC / 10);
 }
 
@@ -304,7 +306,7 @@ QuicCongestionControlOnPersistentCongestionEvent(
         Cc->SlowStartThreshold =
             Cc->CongestionWindow * TEN_TIMES_BETA_CUBIC / 10;
     Cc->CongestionWindow =
-        Connection->Send.PathMtu * QUIC_PERSISTENT_CONGESTION_WINDOW_PACKETS;
+        Connection->Paths[0].Mtu * QUIC_PERSISTENT_CONGESTION_WINDOW_PACKETS;
     Cc->KCubic = 0;
 }
 
@@ -404,7 +406,7 @@ QuicCongestionControlOnDataAcknowledged(
         if (Cc->TimeOfLastAckValid) {
             uint64_t TimeSinceLastAck = QuicTimeDiff64(Cc->TimeOfLastAck, TimeNow);
             if (TimeSinceLastAck > Cc->SendIdleTimeoutMs &&
-                TimeSinceLastAck > US_TO_MS(Connection->SmoothedRtt + 4 * Connection->RttVariance)) {
+                TimeSinceLastAck > US_TO_MS(Connection->Paths[0].SmoothedRtt + 4 * Connection->Paths[0].RttVariance)) {
                 Cc->TimeOfCongAvoidStart += TimeSinceLastAck;
                 if (QuicTimeAtOrBefore64(TimeNow, Cc->TimeOfCongAvoidStart)) {
                     Cc->TimeOfCongAvoidStart = TimeNow;
@@ -438,7 +440,7 @@ QuicCongestionControlOnDataAcknowledged(
 
         int64_t CubicWindow =
             ((((DeltaT * DeltaT) >> 10) * DeltaT *
-              (int64_t)(Connection->Send.PathMtu * TEN_TIMES_C_CUBIC / 10)) >> 20) +
+              (int64_t)(Connection->Paths[0].Mtu * TEN_TIMES_C_CUBIC / 10)) >> 20) +
             (int64_t)Cc->WindowMax;
 
         if (CubicWindow < 0) {
@@ -471,7 +473,7 @@ QuicCongestionControlOnDataAcknowledged(
 
         int64_t AimdWindow =
             Cc->WindowMax * TEN_TIMES_BETA_CUBIC / 10 +
-            TimeInCongAvoid * Connection->Send.PathMtu / (2 * max(1, US_TO_MS(SmoothedRtt)));
+            TimeInCongAvoid * Connection->Paths[0].Mtu / (2 * max(1, US_TO_MS(SmoothedRtt)));
 
         //
         // Use the cubic or AIMD window, whichever is larger.
@@ -487,7 +489,7 @@ QuicCongestionControlOnDataAcknowledged(
             //
             Cc->CongestionWindow +=
                 (uint32_t)max(
-                    ((CubicWindow - Cc->CongestionWindow) * Connection->Send.PathMtu) / Cc->CongestionWindow,
+                    ((CubicWindow - Cc->CongestionWindow) * Connection->Paths[0].Mtu) / Cc->CongestionWindow,
                     1);
         }
     }