Discord DAVE Protocol (E2EE) for voice

.cspell.json

@@ -142,7 +142,12 @@
 		"khanda",
 		"oclock",
 		"moai",
-		"xbps"
+		"xbps",
+		"chacha20",
+		"chacha",
+		"nullopt",
+		"chrono",
+		"ciphersuite"
 	],
 	"flagWords": [
 		"hte"

CMakeLists.txt

@@ -103,6 +103,8 @@ else()
 	add_subdirectory(library)
 endif()
 
+find_package(Filesystem)
+
 if(DPP_USE_EXTERNAL_JSON)
 	# We do nothing here, we just assume it is on the include path.
 	# nlohmann::json's cmake stuff does all kinds of weird, and is more hassle than it's worth.
@@ -116,3 +118,7 @@ else()
 	# that made no sense, it seems they may have changed their parsing rules somehow.
 	message("-- Using bundled nlohmann::json")
 endif()
+
+if (NOT WIN32)
+	target_link_libraries(dpp PRIVATE std::filesystem)
+endif()

README.md

@@ -150,15 +150,17 @@ Other compilers may work (either newer versions of those listed above, or differ
 
 ### External Dependencies (You must install these)
 
-* [OpenSSL](https://openssl.org/) (whichever `-dev` package comes with your OS)
-* [zlib](https://zlib.net) (whichever `-dev` package comes with your OS)
+* [OpenSSL](https://openssl.org/) (For HTTPS, will use whichever `-dev` package comes with your OS)
+* [zlib](https://zlib.net) (For websocket compression, will use whichever `-dev` package comes with your OS)
 
 #### Optional Dependencies
 
-For voice support you require both of:
-* [libopus](https://www.opus-codec.org)
-* [libsodium](https://libsodium.org/)
+For **voice support** you require both of:
+* [libopus](https://www.opus-codec.org) (For audio encoding/decoding)
+* [libsodium](https://libsodium.org/) (For transport layer encryption)
+* Note that our **windows zips** come packaged with copies of both libraries - you do not need to install them yourself!
 
 ### Included Dependencies (Packaged with the library)
 
-* [JSON for Modern C++](https://json.nlohmann.me/)
+* [JSON for Modern C++](https://json.nlohmann.me/) (You can bring your own nlohmann::json into D++ by setting a CMAKE flag)
+* [MLS++](https://github.com/cisco/mlspp) (This is statically compiled into the library if voice support is enabled)

cmake/FindFilesystem.cmake

@@ -0,0 +1,247 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#[=======================================================================[.rst:
+
+FindFilesystem
+##############
+
+This module supports the C++17 standard library's filesystem utilities. Use the
+:imp-target:`std::filesystem` imported target to
+
+Options
+*******
+
+The ``COMPONENTS`` argument to this module supports the following values:
+
+.. find-component:: Experimental
+    :name: fs.Experimental
+
+    Allows the module to find the "experimental" Filesystem TS version of the
+    Filesystem library. This is the library that should be used with the
+    ``std::experimental::filesystem`` namespace.
+
+.. find-component:: Final
+    :name: fs.Final
+
+    Finds the final C++17 standard version of the filesystem library.
+
+If no components are provided, behaves as if the
+:find-component:`fs.Final` component was specified.
+
+If both :find-component:`fs.Experimental` and :find-component:`fs.Final` are
+provided, first looks for ``Final``, and falls back to ``Experimental`` in case
+of failure. If ``Final`` is found, :imp-target:`std::filesystem` and all
+:ref:`variables <fs.variables>` will refer to the ``Final`` version.
+
+
+Imported Targets
+****************
+
+.. imp-target:: std::filesystem
+
+    The ``std::filesystem`` imported target is defined when any requested
+    version of the C++ filesystem library has been found, whether it is
+    *Experimental* or *Final*.
+
+    If no version of the filesystem library is available, this target will not
+    be defined.
+
+    .. note::
+        This target has ``cxx_std_17`` as an ``INTERFACE``
+        :ref:`compile language standard feature <req-lang-standards>`. Linking
+        to this target will automatically enable C++17 if no later standard
+        version is already required on the linking target.
+
+
+.. _fs.variables:
+
+Variables
+*********
+
+.. variable:: CXX_FILESYSTEM_IS_EXPERIMENTAL
+
+    Set to ``TRUE`` when the :find-component:`fs.Experimental` version of C++
+    filesystem library was found, otherwise ``FALSE``.
+
+.. variable:: CXX_FILESYSTEM_HAVE_FS
+
+    Set to ``TRUE`` when a filesystem header was found.
+
+.. variable:: CXX_FILESYSTEM_HEADER
+
+    Set to either ``filesystem`` or ``experimental/filesystem`` depending on
+    whether :find-component:`fs.Final` or :find-component:`fs.Experimental` was
+    found.
+
+.. variable:: CXX_FILESYSTEM_NAMESPACE
+
+    Set to either ``std::filesystem`` or ``std::experimental::filesystem``
+    depending on whether :find-component:`fs.Final` or
+    :find-component:`fs.Experimental` was found.
+
+
+Examples
+********
+
+Using `find_package(Filesystem)` with no component arguments:
+
+.. code-block:: cmake
+
+    find_package(Filesystem REQUIRED)
+
+    add_executable(my-program main.cpp)
+    target_link_libraries(my-program PRIVATE std::filesystem)
+
+
+#]=======================================================================]
+
+
+if(TARGET std::filesystem)
+    # This module has already been processed. Don't do it again.
+    return()
+endif()
+
+cmake_minimum_required(VERSION 3.10)
+
+include(CMakePushCheckState)
+include(CheckIncludeFileCXX)
+
+# If we're not cross-compiling, try to run test executables.
+# Otherwise, assume that compile + link is a sufficient check.
+if(CMAKE_CROSSCOMPILING)
+    include(CheckCXXSourceCompiles)
+    macro(_cmcm_check_cxx_source code var)
+        check_cxx_source_compiles("${code}" ${var})
+    endmacro()
+else()
+    include(CheckCXXSourceRuns)
+    macro(_cmcm_check_cxx_source code var)
+        check_cxx_source_runs("${code}" ${var})
+    endmacro()
+endif()
+
+cmake_push_check_state()
+
+set(CMAKE_REQUIRED_QUIET ${Filesystem_FIND_QUIETLY})
+
+# All of our tests required C++17 or later
+set(CMAKE_CXX_STANDARD 17)
+
+# Normalize and check the component list we were given
+set(want_components ${Filesystem_FIND_COMPONENTS})
+if(Filesystem_FIND_COMPONENTS STREQUAL "")
+    set(want_components Final)
+endif()
+
+# Warn on any unrecognized components
+set(extra_components ${want_components})
+list(REMOVE_ITEM extra_components Final Experimental)
+foreach(component IN LISTS extra_components)
+    message(WARNING "Extraneous find_package component for Filesystem: ${component}")
+endforeach()
+
+# Detect which of Experimental and Final we should look for
+set(find_experimental TRUE)
+set(find_final TRUE)
+if(NOT "Final" IN_LIST want_components)
+    set(find_final FALSE)
+endif()
+if(NOT "Experimental" IN_LIST want_components)
+    set(find_experimental FALSE)
+endif()
+
+if(find_final)
+    check_include_file_cxx("filesystem" _CXX_FILESYSTEM_HAVE_HEADER)
+    mark_as_advanced(_CXX_FILESYSTEM_HAVE_HEADER)
+    if(_CXX_FILESYSTEM_HAVE_HEADER)
+        # We found the non-experimental header. Don't bother looking for the
+        # experimental one.
+        set(find_experimental FALSE)
+    endif()
+else()
+    set(_CXX_FILESYSTEM_HAVE_HEADER FALSE)
+endif()
+
+if(find_experimental)
+    check_include_file_cxx("experimental/filesystem" _CXX_FILESYSTEM_HAVE_EXPERIMENTAL_HEADER)
+    mark_as_advanced(_CXX_FILESYSTEM_HAVE_EXPERIMENTAL_HEADER)
+else()
+    set(_CXX_FILESYSTEM_HAVE_EXPERIMENTAL_HEADER FALSE)
+endif()
+
+if(_CXX_FILESYSTEM_HAVE_HEADER)
+    set(_have_fs TRUE)
+    set(_fs_header filesystem)
+    set(_fs_namespace std::filesystem)
+    set(_is_experimental FALSE)
+elseif(_CXX_FILESYSTEM_HAVE_EXPERIMENTAL_HEADER)
+    set(_have_fs TRUE)
+    set(_fs_header experimental/filesystem)
+    set(_fs_namespace std::experimental::filesystem)
+    set(_is_experimental TRUE)
+else()
+    set(_have_fs FALSE)
+endif()
+
+set(CXX_FILESYSTEM_HAVE_FS ${_have_fs} CACHE BOOL "TRUE if we have the C++ filesystem headers")
+set(CXX_FILESYSTEM_HEADER ${_fs_header} CACHE STRING "The header that should be included to obtain the filesystem APIs")
+set(CXX_FILESYSTEM_NAMESPACE ${_fs_namespace} CACHE STRING "The C++ namespace that contains the filesystem APIs")
+set(CXX_FILESYSTEM_IS_EXPERIMENTAL ${_is_experimental} CACHE BOOL "TRUE if the C++ filesystem library is the experimental version")
+
+set(_found FALSE)
+
+if(CXX_FILESYSTEM_HAVE_FS)
+    # We have some filesystem library available. Do link checks
+    string(CONFIGURE [[
+        #include <cstdlib>
+        #include <@CXX_FILESYSTEM_HEADER@>
+
+        int main() {
+            auto cwd = @CXX_FILESYSTEM_NAMESPACE@::current_path();
+            printf("%s", cwd.c_str());
+            return EXIT_SUCCESS;
+        }
+    ]] code @ONLY)
+
+    # Check a simple filesystem program without any linker flags
+    _cmcm_check_cxx_source("${code}" CXX_FILESYSTEM_NO_LINK_NEEDED)
+
+    set(can_link ${CXX_FILESYSTEM_NO_LINK_NEEDED})
+
+    if(NOT CXX_FILESYSTEM_NO_LINK_NEEDED)
+        set(prev_libraries ${CMAKE_REQUIRED_LIBRARIES})
+        # Add the libstdc++ flag
+        set(CMAKE_REQUIRED_LIBRARIES ${prev_libraries} -lstdc++fs)
+        _cmcm_check_cxx_source("${code}" CXX_FILESYSTEM_STDCPPFS_NEEDED)
+        set(can_link ${CXX_FILESYSTEM_STDCPPFS_NEEDED})
+        if(NOT CXX_FILESYSTEM_STDCPPFS_NEEDED)
+            # Try the libc++ flag
+            set(CMAKE_REQUIRED_LIBRARIES ${prev_libraries} -lc++fs)
+            _cmcm_check_cxx_source("${code}" CXX_FILESYSTEM_CPPFS_NEEDED)
+            set(can_link ${CXX_FILESYSTEM_CPPFS_NEEDED})
+        endif()
+    endif()
+
+    if(can_link)
+        add_library(std::filesystem INTERFACE IMPORTED)
+        set_property(TARGET std::filesystem APPEND PROPERTY INTERFACE_COMPILE_FEATURES cxx_std_17)
+        set(_found TRUE)
+
+        if(CXX_FILESYSTEM_NO_LINK_NEEDED)
+            # Nothing to add...
+        elseif(CXX_FILESYSTEM_STDCPPFS_NEEDED)
+            set_property(TARGET std::filesystem APPEND PROPERTY INTERFACE_LINK_LIBRARIES -lstdc++fs)
+        elseif(CXX_FILESYSTEM_CPPFS_NEEDED)
+            set_property(TARGET std::filesystem APPEND PROPERTY INTERFACE_LINK_LIBRARIES -lc++fs)
+        endif()
+    endif()
+endif()
+
+cmake_pop_check_state()
+
+set(Filesystem_FOUND ${_found} CACHE BOOL "TRUE if we can run a program using std::filesystem" FORCE)
+
+if(Filesystem_FIND_REQUIRED AND NOT Filesystem_FOUND)
+    message(FATAL_ERROR "Cannot run simple program using std::filesystem")
+endif()

docpages/advanced_reference/unit_tests.md

@@ -24,6 +24,13 @@ export TEST_USER_ID="826535422381391913"
 export TEST_EVENT_ID="909928577951203360"
 ```
 
+You may also optionally set:
+```bash
+export TEST_DATA_DIR="/path/to/test/data"
+```
+If you wish to have test data (Robot.pcm etc) in a different location than two directories above the unit test program. If you do not specify
+this environment variable the default will be used.
+
 Then, after cloning and building DPP, run `cd build && ctest -VV` for unit test cases. 
 
 If you do not specify the `DPP_UNIT_TEST_TOKEN` environment variable, a subset of the tests will run which do not require discord connectivity.

docpages/advanced_reference/voice_model.md

@@ -1,5 +1,42 @@
 \page voice-model Voice Model
 
+# High Level Summary
+
+Discord's audio system consists of several layers and inter-related systems as shown in the flow chart below.
+
+At the top level, connecting to a voice server requires the library to request the details of a voice server from Discord via the websocket for the shard where the
+server is located. Performing this request will make Discord reply with a websocket URI and an ephemeral token (not the bot token) which are used to establish an
+initial connection to this secondary websocket. Every connection to a voice channel creates a separate secondary websocket.
+
+Once connected to this websocket, the library negotiates which protocols it supports and what encryption schemes to use. If you enabled DAVE (Discord's end-to-end
+encryption scheme) this is negotiated first. An MLS (message layer security) group is joined or created. If you did not enable DAVE, this step is bypassed.
+
+The secondary websocket then gives the library a shared encryption secret and the hostname of an RTP server, which is used to encrypt RTP packets using libsodium.
+This is stored for later.
+
+The next step is to send an initial packet to the RTP server so that the library can detect the public IP where the bot is running. Once the RTP server replies,
+the bot may tell the websocket what encryption protocols it is going to use to encrypt the RTP packet contents (leaving the RTP header somwhat intact).
+
+The library is now in an initialised state and will accept method calls for `send_audio_raw()` and `send_audio_opus()`. If you send raw audio, it will first be
+encoded as OPUS using libopus, and potentially repacketized to fit into UDP packets, with larger streams being split into multiple smaller packets that are scheduled
+to be sent in the future.
+
+If at this point DAVE is enabled, the contents of the OPUS encoded audio are encrypted using the AES 128 bit AEAD cipher and using the bot's MLS ratchet, derived
+during the MLS negotiation which was carried out earlier. All valid participants in the voice channel may use their private key, and the public key derived from
+their ratchets, to decrypt the OPUS audio.
+
+Regardless of if DAVE is enabled or not, the OPUS stream (encrypted by DAVE, or "plaintext") is placed into an RTP packet, and then encrypted using the shared secret
+given by the websocket, known only to you and Discord, using the xchacha20 poly1305 cipher.
+
+The completed packet, potentially with two separate layers of encryption (one with a key only you and Discord know, and one with a key only you and participants in the
+voice chat know!), plus opus encoded audio is sent on its way via UDP to the RTP server, where Discord promptly distribute it to all participants in the chat.
+
+\image html audioframe.svg
+
+After reading all this, go get a coffee or something, you deserve it! ☕
+
+# Flow Diagram
+
 \dot
 digraph "Example Directory" {
 	graph [ranksep=1];
@@ -109,6 +146,26 @@ digraph "Example Directory" {
 		"HTTP/1.1 101 Switching Protocols" -> "discord_voice_client::handle_frame";
 
 		label = "Do the voice stuff.";
+
+        "discord_voice_client::handle_frame"-> "DAVE enabled";
+        "discord_voice_client::handle_frame"-> "DAVE disabled";
+        "DAVE disabled"->"discord_voice_client::send_audio_*()";
+ "discord_voice_client::send_audio_*()" -> "Dave encryption on";
+ "discord_voice_client::send_audio_*()" -> "Dave encryption off";
+ "Dave encryption on" -> "AES AEAD encryption\nof OPUS stream\nusing ratchet";
+ "AES AEAD encryption\nof OPUS stream\nusing ratchet" -> "SODIUM encryption (xchacha20_poly1305_aead)";
+  "Dave encryption off" -> "SODIUM encryption (xchacha20_poly1305_aead)";
+ "SODIUM encryption (xchacha20_poly1305_aead)" -> "UDP sendto";
+ "UDP sendto" -> "Discord RTP server";
+ "DAVE enabled" -> "MLS send key package";
+ "MLS send key package" -> "MLS receive external sender";
+ "MLS receive external sender" -> "MLS proposals";
+ "MLS proposals" -> "MLS Welcome";
+ "MLS proposals" -> "MLS Commit";
+ "MLS Commit" -> "DAVE begin transition";
+ "MLS Welcome" -> "DAVE begin transition";
+ "DAVE begin transition" -> "Dave execute transition";
+ "Dave execute transition" -> "discord_voice_client::send_audio_*()";
 	}
 
 	"Your bot" -> "guild::connect_member_voice";