Merge branch 'main' into partitions_test_uses_same_settings

achamayou · Jun 28, 2024 · 9ba615c · 9ba615c
2 parents 9078734 + 3db1171
commit 9ba615c
Show file tree

Hide file tree

Showing 82 changed files with 534 additions and 473 deletions.
diff --git a/.azure-pipelines-templates/test.yml b/.azure-pipelines-templates/test.yml
@@ -7,8 +7,6 @@ parameters:
 steps:
   - script: |
       set -ex
-      df -h /dev/shm
-      du -sh /dev/shm/*
       ASAN_SYMBOLIZER=/usr/bin/llvm-symbolizer-15
       export ASAN_SYMBOLIZER_PATH=$(realpath ${ASAN_SYMBOLIZERS})
       ./tests.sh -VV --timeout ${{ parameters.ctest_timeout }} --no-compress-output -T Test ${{ parameters.ctest_filter }}

diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
@@ -31,15 +31,15 @@ resources:
   containers:
     - container: virtual
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: snp
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: sgx
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro
 
 variables:
   perf_or_release: perf

diff --git a/.daily.yml b/.daily.yml
@@ -28,15 +28,15 @@ resources:
   containers:
     - container: virtual
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
 
     - container: snp
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: sgx
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx
 
 jobs:
   - template: .azure-pipelines-templates/configure.yml

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -19,15 +19,15 @@ jobs:
           - name: virtual
             image: default
             nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
-            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
           - name: snp
             image: default
             nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
-            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
           - name: sgx
             image: sgx
             nodes: [self-hosted, 1ES.Pool=gha-sgx-ccf-sub]
-            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro
     runs-on: ${{ matrix.platform.nodes }}
     container:
       image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-26-06-2024
@@ -68,8 +68,6 @@ jobs:
           cd build
           rm -rf /github/home/.cache
           mkdir -p /github/home/.cache
-          rm -rf /dev/shm/*
-          df -h /dev/shm
           export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15)
           ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|protocolstest|vegeta|suite"
         shell: bash

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,8 +17,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   - `::threading` is now `ccf::threading`, and `ccf/ds/thread_ids.h` has moved to `ccf/threading/thread_ids.h`
   - `::consensus` is now `ccf::consensus`
   - `::tls` is now `ccf::tls`
+  - `::http` is now `ccf::http`
   - `::nonstd` is now `ccf::nonstd`
   - `::crypto` is now `ccf::crypto`
+  - `::logger` is now `ccf::logger`
 - The `programmability` sample app now demonstrates how applications can define their own extensions, creating bindings between C++ and JS state, and allowing JS endpoints to call functions implemented in C++.
 - Introduce `DynamicJSEndpointRegistry::record_action_for_audit_v1` and `DynamicJSEndpointRegistry::check_action_not_replayed_v1` to allow an application making use of the programmability feature to easily implement auditability, and protect users allowed to update the application against replay attacks (#6285).
 - Endpoints now support a `ToBackup` redirection strategy, for requests which should never be executed on a primary. These must also be read-only. These are configured similar to `ToPrimary` endpoints, with a `to_backup` object (specifying by-role or statically-addressed targets) in each node's configuration.

diff --git a/include/ccf/ds/logger.h b/include/ccf/ds/logger.h
@@ -15,7 +15,7 @@
 #include <sstream>
 #include <type_traits>
 
-namespace logger
+namespace ccf::logger
 {
   static constexpr LoggerLevel MOST_VERBOSE =
 #ifdef CCF_DISABLE_VERBOSE_LOGGING
@@ -356,9 +356,9 @@ namespace logger
 // This allows:
 // CCF_LOG_OUT(DEBUG, "foo") << "this " << "msg";
 #define CCF_LOG_OUT(LVL, TAG) \
-  logger::config::ok(LoggerLevel::LVL) && \
-    logger::Out() == \
-      logger::LogLine(LoggerLevel::LVL, TAG, __FILE__, __LINE__)
+  ccf::logger::config::ok(LoggerLevel::LVL) && \
+    ccf::logger::Out() == \
+      ccf::logger::LogLine(LoggerLevel::LVL, TAG, __FILE__, __LINE__)
 
 // To avoid repeating the (s, ...) args for every macro, we cheat with a curried
 // macro here by ending the macro with another macro name, which then accepts
@@ -376,7 +376,7 @@ namespace logger
   };
 
 #ifndef CCF_LOGGER_NO_DEPRECATE
-#  define CCF_LOGGER_DEPRECATE(MACRO) logger::macro::MACRO;
+#  define CCF_LOGGER_DEPRECATE(MACRO) ccf::logger::macro::MACRO;
 #else
 #  define CCF_LOGGER_DEPRECATE(MACRO)
 #endif

diff --git a/include/ccf/http_configuration.h b/include/ccf/http_configuration.h
@@ -7,7 +7,7 @@
 
 #include <optional>
 
-namespace http
+namespace ccf::http
 {
   // Default parser limits, used as a DoS protection against
   // requests that are too large.

diff --git a/include/ccf/http_consts.h b/include/ccf/http_consts.h
@@ -2,72 +2,75 @@
 // Licensed under the Apache 2.0 License.
 #pragma once
 
-namespace http
+namespace ccf
 {
-  namespace headers
+  namespace http
   {
-    // All HTTP headers are expected to be lowercase
-    static constexpr auto ACCEPT = "accept";
-    static constexpr auto ALLOW = "allow";
-    static constexpr auto AUTHORIZATION = "authorization";
-    static constexpr auto CACHE_CONTROL = "cache-control";
-    static constexpr auto CONTENT_LENGTH = "content-length";
-    static constexpr auto CONTENT_TYPE = "content-type";
-    static constexpr auto DATE = "date";
-    static constexpr auto DIGEST = "digest";
-    static constexpr auto HOST = "host";
-    static constexpr auto LOCATION = "location";
-    static constexpr auto RETRY_AFTER = "retry-after";
-    static constexpr auto TRAILER = "trailer";
-    static constexpr auto WWW_AUTHENTICATE = "www-authenticate";
+    namespace headers
+    {
+      // All HTTP headers are expected to be lowercase
+      static constexpr auto ACCEPT = "accept";
+      static constexpr auto ALLOW = "allow";
+      static constexpr auto AUTHORIZATION = "authorization";
+      static constexpr auto CACHE_CONTROL = "cache-control";
+      static constexpr auto CONTENT_LENGTH = "content-length";
+      static constexpr auto CONTENT_TYPE = "content-type";
+      static constexpr auto DATE = "date";
+      static constexpr auto DIGEST = "digest";
+      static constexpr auto HOST = "host";
+      static constexpr auto LOCATION = "location";
+      static constexpr auto RETRY_AFTER = "retry-after";
+      static constexpr auto TRAILER = "trailer";
+      static constexpr auto WWW_AUTHENTICATE = "www-authenticate";
 
-    static constexpr auto CCF_TX_ID = "x-ms-ccf-transaction-id";
-  }
+      static constexpr auto CCF_TX_ID = "x-ms-ccf-transaction-id";
+    }
 
-  namespace headervalues
-  {
-    namespace contenttype
+    namespace headervalues
     {
-      static constexpr auto JSON = "application/json";
-      static constexpr auto MSGPACK = "application/msgpack";
-      static constexpr auto TEXT = "text/plain";
-      static constexpr auto OCTET_STREAM = "application/octet-stream";
-      static constexpr auto GRPC = "application/grpc";
-      static constexpr auto COSE = "application/cose";
-      static constexpr auto JAVASCRIPT = "text/javascript";
+      namespace contenttype
+      {
+        static constexpr auto JSON = "application/json";
+        static constexpr auto MSGPACK = "application/msgpack";
+        static constexpr auto TEXT = "text/plain";
+        static constexpr auto OCTET_STREAM = "application/octet-stream";
+        static constexpr auto GRPC = "application/grpc";
+        static constexpr auto COSE = "application/cose";
+        static constexpr auto JAVASCRIPT = "text/javascript";
+      }
     }
-  }
 
-  namespace auth
-  {
-    static constexpr auto DIGEST_SHA256 = "SHA-256";
+    namespace auth
+    {
+      static constexpr auto DIGEST_SHA256 = "SHA-256";
 
-    static constexpr auto SIGN_AUTH_SCHEME = "Signature";
-    static constexpr auto SIGN_PARAMS_KEYID = "keyId";
-    static constexpr auto SIGN_PARAMS_SIGNATURE = "signature";
-    static constexpr auto SIGN_PARAMS_ALGORITHM = "algorithm";
-    static constexpr auto SIGN_PARAMS_HEADERS = "headers";
-    static constexpr auto SIGN_ALGORITHM_ECDSA_SHA256 = "ecdsa-sha256";
-    static constexpr auto SIGN_ALGORITHM_HS_2019 = "hs2019";
+      static constexpr auto SIGN_AUTH_SCHEME = "Signature";
+      static constexpr auto SIGN_PARAMS_KEYID = "keyId";
+      static constexpr auto SIGN_PARAMS_SIGNATURE = "signature";
+      static constexpr auto SIGN_PARAMS_ALGORITHM = "algorithm";
+      static constexpr auto SIGN_PARAMS_HEADERS = "headers";
+      static constexpr auto SIGN_ALGORITHM_ECDSA_SHA256 = "ecdsa-sha256";
+      static constexpr auto SIGN_ALGORITHM_HS_2019 = "hs2019";
 
-    static constexpr auto SIGN_HEADER_REQUEST_TARGET = "(request-target)";
+      static constexpr auto SIGN_HEADER_REQUEST_TARGET = "(request-target)";
 
-    static constexpr auto SIGN_PARAMS_DELIMITER = ",";
-    static constexpr auto SIGN_PARAMS_HEADERS_DELIMITER = " ";
+      static constexpr auto SIGN_PARAMS_DELIMITER = ",";
+      static constexpr auto SIGN_PARAMS_HEADERS_DELIMITER = " ";
 
-    static constexpr auto BEARER_AUTH_SCHEME = "Bearer";
-  }
+      static constexpr auto BEARER_AUTH_SCHEME = "Bearer";
+    }
 
-  static constexpr char const* required_signature_headers[] = {
-    auth::SIGN_HEADER_REQUEST_TARGET, headers::DIGEST};
-}
+    static constexpr char const* required_signature_headers[] = {
+      auth::SIGN_HEADER_REQUEST_TARGET, ccf::http::headers::DIGEST};
+  }
 
-namespace http2
-{
-  namespace headers
+  namespace http2
   {
-    static constexpr auto PATH = ":path";
-    static constexpr auto STATUS = ":status";
-    static constexpr auto METHOD = ":method";
+    namespace headers
+    {
+      static constexpr auto PATH = ":path";
+      static constexpr auto STATUS = ":status";
+      static constexpr auto METHOD = ":method";
+    }
   }
 }
diff --git a/include/ccf/http_header_map.h b/include/ccf/http_header_map.h
@@ -5,7 +5,7 @@
 #include <map>
 #include <string>
 
-namespace http
+namespace ccf::http
 {
   using HeaderMap = std::map<std::string, std::string, std::less<>>;
   using HeaderKeyValue = HeaderMap::value_type;

diff --git a/include/ccf/http_query.h b/include/ccf/http_query.h
@@ -11,7 +11,7 @@
 #include <optional>
 #include <string_view>
 
-namespace http
+namespace ccf::http
 {
   // Query is parsed into a multimap, so that duplicate keys are retained.
   // Handling of duplicates (or ignoring them entirely) is left to the caller.

diff --git a/include/ccf/http_responder.h b/include/ccf/http_responder.h
@@ -8,7 +8,7 @@
 
 #include <functional>
 
-namespace http
+namespace ccf::http
 {
   using StreamOnCloseCallback = std::function<void(void)>;
 

diff --git a/include/ccf/node/rpc_context_impl.h b/include/ccf/node/rpc_context_impl.h
@@ -78,7 +78,7 @@ namespace ccf
       std::string&& msg,
       const std::vector<nlohmann::json>& details = {}) override
     {
-      auto content_type = get_request_header(http::headers::CONTENT_TYPE);
+      auto content_type = get_request_header(ccf::http::headers::CONTENT_TYPE);
       if (
         content_type.has_value() &&
         content_type.value() == http::headervalues::contenttype::GRPC)
@@ -110,7 +110,8 @@ namespace ccf
       set_response_status(status);
       set_response_body(std::vector<uint8_t>(s.begin(), s.end()));
       set_response_header(
-        http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON);
+        ccf::http::headers::CONTENT_TYPE,
+        http::headervalues::contenttype::JSON);
     }
 
     void set_grpc_error(grpc_status grpc_status, std::string&& msg)
@@ -122,7 +123,8 @@ namespace ccf
 
       set_response_status(HTTP_STATUS_OK);
       set_response_header(
-        http::headers::CONTENT_TYPE, http::headervalues::contenttype::GRPC);
+        ccf::http::headers::CONTENT_TYPE,
+        http::headervalues::contenttype::GRPC);
       set_response_trailer(grpc::make_status_trailer(grpc_status));
       set_response_trailer(grpc::make_message_trailer(msg));
     }

diff --git a/include/ccf/research/grpc_status.h b/include/ccf/research/grpc_status.h
@@ -97,7 +97,7 @@ static grpc_status http_status_to_grpc(http_status s)
 namespace ccf::grpc
 {
   static const http::HeaderMap default_response_headers = {
-    {http::headers::CONTENT_TYPE, http::headervalues::contenttype::GRPC}};
+    {ccf::http::headers::CONTENT_TYPE, http::headervalues::contenttype::GRPC}};
 
   static constexpr auto TRAILER_STATUS = "grpc-status";
   static constexpr auto TRAILER_MESSAGE = "grpc-message";

diff --git a/include/ccf/rpc_context.h b/include/ccf/rpc_context.h
@@ -80,7 +80,7 @@ namespace ccf
     virtual const ccf::RESTVerb& get_request_verb() const = 0;
     virtual std::string get_request_path() const = 0;
     virtual std::string get_method() const = 0;
-    virtual std::shared_ptr<http::HTTPResponder> get_responder() const = 0;
+    virtual std::shared_ptr<ccf::http::HTTPResponder> get_responder() const = 0;
 
     /// Returns a map of all PathParams parsed out of the original query path.
     /// For instance if this endpoint was installed at `/foo/{name}/{age}`, and

diff --git a/samples/apps/basic/basic.cpp b/samples/apps/basic/basic.cpp
@@ -74,7 +74,8 @@ namespace basicapp
         {
           ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK);
           ctx.rpc_ctx->set_response_header(
-            http::headers::CONTENT_TYPE, http::headervalues::contenttype::TEXT);
+            ccf::http::headers::CONTENT_TYPE,
+            ccf::http::headervalues::contenttype::TEXT);
           ctx.rpc_ctx->set_response_body(record.value());
           return;
         }