Add service identity endorsements to historical receipts (#3679)

.daily_canary

@@ -1 +1 @@
-Run!!!!
+There, he moved!

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - The `enclave::` namespace has been removed, and all types which were under it are now under `ccf::`. This will affect any apps using `enclave::RpcContext`, which should be replaced with `ccf::RpcContext` (#3664).
 - HTTP parsing errors are now recorded per-interface and returned by `GET /node/metrics` (#3671).
 - The `kv::Store` type is no longer visible to application code, and is replaced by a simpler `kv::ReadOnlyStore`. This is the interface given to historical queries to access historical state and enforces read-only access, without exposing internal implementation details of the store. This should have no impact on JS apps, but C++ apps will need to replace calls to `store->current_txid()` with calls to `store->get_txid()`, and `store->create_tx()` to `store->create_read_only_tx()`.
+- Receipts now come with service endorsements of previous service identities after recoveries (#3679). See `verify_receipt` in `e2e_logging.py` for an example of how to verify the resulting certificate chain. This functionality is introduced in `ccf::historical::adapter_v3`.
 
 ## [2.0.0-rc4]
 

doc/schemas/app_openapi.json

@@ -201,6 +201,15 @@
         "pattern": "^[a-f0-9]{64}$",
         "type": "string"
       },
+      "Pem": {
+        "type": "string"
+      },
+      "Pem_array": {
+        "items": {
+          "$ref": "#/components/schemas/Pem"
+        },
+        "type": "array"
+      },
       "Receipt": {
         "properties": {
           "cert": {
@@ -221,6 +230,9 @@
           "root": {
             "$ref": "#/components/schemas/string"
           },
+          "service_endorsements": {
+            "$ref": "#/components/schemas/Pem_array"
+          },
           "signature": {
             "$ref": "#/components/schemas/string"
           }
@@ -310,7 +322,7 @@
   "info": {
     "description": "This CCF sample app implements a simple logging application, securely recording messages at client-specified IDs. It demonstrates most of the features available to CCF apps.",
     "title": "CCF Sample Logging App",
-    "version": "1.9.0"
+    "version": "1.9.1"
   },
   "openapi": "3.0.0",
   "paths": {

doc/schemas/gov_openapi.json

@@ -252,6 +252,12 @@
       "Pem": {
         "type": "string"
       },
+      "Pem_array": {
+        "items": {
+          "$ref": "#/components/schemas/Pem"
+        },
+        "type": "array"
+      },
       "Proposal": {
         "properties": {
           "actions": {
@@ -354,6 +360,9 @@
           "root": {
             "$ref": "#/components/schemas/string"
           },
+          "service_endorsements": {
+            "$ref": "#/components/schemas/Pem_array"
+          },
           "signature": {
             "$ref": "#/components/schemas/string"
           }
@@ -472,7 +481,7 @@
   "info": {
     "description": "This API is used to submit and query proposals which affect CCF's public governance tables.",
     "title": "CCF Governance API",
-    "version": "2.7.0"
+    "version": "2.7.1"
   },
   "openapi": "3.0.0",
   "paths": {

doc/schemas/node_openapi.json

@@ -498,6 +498,12 @@
       "Pem": {
         "type": "string"
       },
+      "Pem_array": {
+        "items": {
+          "$ref": "#/components/schemas/Pem"
+        },
+        "type": "array"
+      },
       "Quote": {
         "properties": {
           "endorsements": {
@@ -556,6 +562,9 @@
           "root": {
             "$ref": "#/components/schemas/string"
           },
+          "service_endorsements": {
+            "$ref": "#/components/schemas/Pem_array"
+          },
           "signature": {
             "$ref": "#/components/schemas/string"
           }
@@ -773,7 +782,7 @@
   "info": {
     "description": "This API provides public, uncredentialed access to service and node state.",
     "title": "CCF Public Node API",
-    "version": "2.16.0"
+    "version": "2.16.1"
   },
   "openapi": "3.0.0",
   "paths": {

doc/use_apps/verify_tx.rst

@@ -77,6 +77,19 @@ This means that the request may return ``202 Accepted`` at first, with a suggest
               {'right': '8e238d95767e6ffe4b20e1a5e93dd7b926cbd86caa83698584a16ad2dd7d60b8'},
               {'left': 'd4717996ae906cdce0ac47257a4a9445c58474c2f40811e575f804506e5fee9f'},
               {'left': 'c1c206c4670bd2adee821013695d593f5983ca0994ae74630528da5fb6642205'}],
+    'service_endorsements': [ '-----BEGIN CERTIFICATE-----'
+                            'MIIBtTCCATugAwIBAgIRAN37fxGnWYNVLZn8nM8iBP8wCgYIKoZIzj0EAwMwFjEU\n'
+                            'MBIGA1UEAwwLQ0NGIE5ldHdvcmswHhcNMjIwMzIzMTMxMDA2WhcNMjIwMzI0MTMx\n'
+                            'MDA1WjAWMRQwEgYDVQQDDAtDQ0YgTmV0d29yazB2MBAGByqGSM49AgEGBSuBBAAi\n'
+                            'A2IABBErIfAEVg2Uw+iBPV9kEcpQw8NcoZWHmj4boHf7VVd6yCwRl+X/wOaOudca\n'
+                            'CqMMcwrt4Bb7n11RbsRwU04B7fG907MelICFHiPZjU/XMK5HEsSEZWowVtNwOLDo\n'
+                            'l5cN6aNNMEswCQYDVR0TBAIwADAdBgNVHQ4EFgQU4n5gHhHFnYZc3nwxKRggl8YB\n'
+                            'qdgwHwYDVR0jBBgwFoAUcAvR3F5YSUvPPGcAxrvh2Z5ump8wCgYIKoZIzj0EAwMD\n'
+                            'aAAwZQIxAMeRoXo9FDzr51qkiD4Ws0Y+KZT06MFHcCg47TMDSGvnGrwL3DcIjGs7\n'
+                            'TTwJJQjbWAIwS9AqOJP24sN6jzXOTd6RokeF/MTGJbQAihzgTbZia7EKM8s/0yDB\n'
+                            '0QYtrfMjtPOx\n'
+                            '-----END CERTIFICATE-----\n'
+    ],
     'signature': 'MGQCMHrnwS123oHqUKuQRPsQ+gk6WVutixeOvxcXX79InBgPOxJCoScCOlBnK4UYyLzangIwW9k7IZkMgG076qVv5zcx7OuKb7bKyii1yP1rcakeGVvVMwISeE+Fr3BnFfPD66Df'}
 
 `cert` contains the certificate of the signing node, endorsed by the service identity. `node_id` is the node's ID inside CCF, a digest of its public key.
@@ -97,6 +110,8 @@ The proof is empty, and the ``leaf`` field is set to the value being signed, whi
 This allows writing verification code that handles both regular and signature receipts similarly, but it is worth noting that the 'leaf' value for signatures is _not_
 the digest of the signature transaction itself.
 
+From version 2.0, CCF also includes endorsement certificates for previous service identities, by the current service identity, in `service_endorsements`. Thus, after at least one recovery, the endorsement check now takes the form of a certificate chain verification instead of a single endorsement check.
+
 Receipt Verification
 --------------------
 
@@ -106,7 +121,7 @@ Verifying a receipt consists of the following steps:
   2. If the receipt contains ``leaf_components``, digest the concatenation ``write_set_digest + commit_evidence_digest + claims_digest`` to produce ``leaf``.
   3. Combine ``leaf`` with the successive elements in ``proof`` to calculate the value of ``root``. See :py:func:`ccf.receipt.root` for a reference implementation.
   4. Verify ``signature`` over the ``root`` using the certificate of the node identified by ``node_id`` and ``cert``. See :py:func:`ccf.receipt.verify` for a reference implementation.
-  5. Check that the certificate ``cert`` of ``node_id`` used to sign the receipt is endorsed by the CCF network. See :py:func:`ccf.receipt.check_endorsement` for a reference implementation.
+  5. Check that the certificate ``cert`` of ``node_id`` used to sign the receipt is endorsed by the CCF network. See :py:func:`ccf.receipt.check_endorsements` for a reference implementation.
 
 Note that since a receipt is a committment by a service to a transaction, a verifier must know the service identity, and provide it as an input to step 5.
 

include/ccf/crypto/key_pair.h

@@ -48,19 +48,38 @@ namespace crypto
 
     virtual Pem create_csr(
       const std::string& subject_name,
-      const std::vector<SubjectAltName>& subject_alt_names) const = 0;
+      const std::vector<SubjectAltName>& subject_alt_names,
+      const std::optional<Pem>& public_key = std::nullopt) const = 0;
 
     Pem create_csr(const std::string& subject_name) const
     {
       return create_csr(subject_name, {});
     }
 
+    // Note about the signed_by_issuer parameter to sign_csr: when issuing a new
+    // certificate for an old subject, which does not exist anymore, we cannot
+    // sign the CSR with that old subject's private key. Instead, the issuer
+    // signs the CSR itself, which is slightly unusal. Instead, we could also
+    // ask the subject to produce a CSR right after it becomes alive and keep it
+    // around until we need it, but those complications are not stricly
+    // necessary. In our case, we use this to re-endorse previous service
+    // identities, which are self-signed, and replace them with new endorsements
+    // by the current service identity (which doesn't have the private key of
+    // previous ones).
+
+    enum class Signer
+    {
+      SUBJECT = 0,
+      ISSUER = 1
+    };
+
     virtual Pem sign_csr(
       const Pem& issuer_cert,
       const Pem& signing_request,
       const std::string& valid_from,
       const std::string& valid_to,
-      bool ca = false) const = 0;
+      bool ca = false,
+      Signer signer = Signer::SUBJECT) const = 0;
 
     Pem self_sign(
       const std::string& name,

include/ccf/historical_queries_adapter.h

@@ -5,6 +5,7 @@
 #include "ccf/ccf_deprecated.h"
 #include "ccf/endpoint_context.h"
 #include "ccf/historical_queries_interface.h"
+#include "ccf/node_context.h"
 #include "ccf/tx_id.h"
 #include "ccf/tx_status.h"
 
@@ -49,9 +50,25 @@ namespace ccf::historical
     ccf::SeqNo seqno,
     std::string& error_reason);
 
+  ccf::endpoints::EndpointFunction adapter_v3(
+    const HandleHistoricalQuery& f,
+    ccfapp::AbstractNodeContext& node_context,
+    const CheckHistoricalTxStatus& available,
+    const TxIDExtractor& extractor = txid_from_header);
+
+  /// @cond
+  // Doxygen cannot parse these declarations; some combination of a macro,
+  // attribute syntax, and namespaced types results in the following warning
+  // (treated as error):
+  //   Found ';' while parsing initializer list! (doxygen could be confused by a
+  //   macro call without semicolon)
+  // Use label-less cond to unconditionally exclude this block from parsing
+  // until the declarations are removed are removed.
+  CCF_DEPRECATED(
+    "Will be removed in 3.0, switch to ccf::historical::adapter_v3")
   ccf::endpoints::EndpointFunction adapter_v2(
     const HandleHistoricalQuery& f,
-    AbstractStateCache& state_cache,
+    ccfapp::AbstractNodeContext& node_context,
     const CheckHistoricalTxStatus& available,
     const TxIDExtractor& extractor = txid_from_header);
 

include/ccf/receipt.h

@@ -3,9 +3,12 @@
 
 #pragma once
 
+#include "ccf/crypto/pem.h"
 #include "ccf/ds/json.h"
 #include "ccf/entity_id.h"
 
+#include <optional>
+
 namespace ccf
 {
   struct Receipt
@@ -51,6 +54,8 @@ namespace ccf
     std::optional<std::string> leaf = std::nullopt;
     /// Leaf components in transactions emitted by 2.x networks.
     std::optional<LeafComponents> leaf_components = std::nullopt;
+
+    std::optional<std::vector<crypto::Pem>> service_endorsements = std::nullopt;
   };
 
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(Receipt::Element)
@@ -64,5 +69,6 @@ namespace ccf
 
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(Receipt)
   DECLARE_JSON_REQUIRED_FIELDS(Receipt, signature, proof, node_id)
-  DECLARE_JSON_OPTIONAL_FIELDS(Receipt, root, cert, leaf, leaf_components)
+  DECLARE_JSON_OPTIONAL_FIELDS(
+    Receipt, root, cert, leaf, leaf_components, service_endorsements)
 }

include/ccf/service/tables/service.h

@@ -2,6 +2,7 @@
 // Licensed under the Apache 2.0 License.
 #pragma once
 
+#include "ccf/crypto/pem.h"
 #include "ccf/ds/json.h"
 #include "ccf/service/map.h"
 
@@ -27,13 +28,13 @@ namespace ccf
     /// x.509 Service Certificate, as a PEM string
     crypto::Pem cert;
     /// Status of the service
-    ServiceStatus status;
-    /// Previous service identity, before the last recovery
-    std::optional<crypto::Pem> previous_service_identity = std::nullopt;
+    ServiceStatus status = ServiceStatus::OPENING;
+    /// Version of previous service identity (before the last recovery)
+    std::optional<kv::Version> previous_service_identity_version = std::nullopt;
   };
-  DECLARE_JSON_TYPE(ServiceInfo);
+  DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(ServiceInfo);
   DECLARE_JSON_REQUIRED_FIELDS(ServiceInfo, cert, status);
-  DECLARE_JSON_OPTIONAL_FIELDS(ServiceInfo, previous_service_identity);
+  DECLARE_JSON_OPTIONAL_FIELDS(ServiceInfo, previous_service_identity_version);
 
   // As there is only one service active at a given time, it is stored in single
   // Value in the KV

python/ccf/receipt.py

@@ -51,3 +51,13 @@ def check_endorsement(endorsee: Certificate, endorser: Certificate):
     endorser_pk.verify(
         endorsee.signature, digest, ec.ECDSA(utils.Prehashed(digest_algo))
     )
+
+
+def check_endorsements(
+    node_cert: Certificate, service_cert: Certificate, endorsements: List[Certificate]
+):
+    cert_i = node_cert
+    for endorsement in endorsements:
+        check_endorsement(cert_i, endorsement)
+        cert_i = endorsement
+    check_endorsement(cert_i, service_cert)

samples/apps/logging/logging.cpp

@@ -222,7 +222,7 @@ namespace loggingapp
         "This CCF sample app implements a simple logging application, securely "
         "recording messages at client-specified IDs. It demonstrates most of "
         "the features available to CCF apps.";
-      openapi_info.document_version = "1.9.0";
+      openapi_info.document_version = "1.9.1";
 
       index_per_public_key = std::make_shared<RecordsIndexingStrategy>(
         PUBLIC_RECORDS, context, 10000, 20);
@@ -842,8 +842,7 @@ namespace loggingapp
       make_endpoint(
         "/log/private/historical",
         HTTP_GET,
-        ccf::historical::adapter_v2(
-          get_historical, context.get_historical_state(), is_tx_committed),
+        ccf::historical::adapter_v3(get_historical, context, is_tx_committed),
         auth_policies)
         .set_auto_schema<void, LoggingGetHistorical::Out>()
         .add_query_parameter<size_t>("id")
@@ -896,10 +895,8 @@ namespace loggingapp
       make_endpoint(
         "/log/private/historical_receipt",
         HTTP_GET,
-        ccf::historical::adapter_v2(
-          get_historical_with_receipt,
-          context.get_historical_state(),
-          is_tx_committed),
+        ccf::historical::adapter_v3(
+          get_historical_with_receipt, context, is_tx_committed),
         auth_policies)
         .set_auto_schema<void, LoggingGetReceipt::Out>()
         .add_query_parameter<size_t>("id")
@@ -956,10 +953,8 @@ namespace loggingapp
       make_endpoint(
         "/log/public/historical_receipt",
         HTTP_GET,
-        ccf::historical::adapter_v2(
-          get_historical_with_receipt_and_claims,
-          context.get_historical_state(),
-          is_tx_committed),
+        ccf::historical::adapter_v3(
+          get_historical_with_receipt_and_claims, context, is_tx_committed),
         auth_policies)
         .set_auto_schema<void, LoggingGetReceipt::Out>()
         .add_query_parameter<size_t>("id")

src/apps/js_generic/js_generic_base.cpp

@@ -245,7 +245,7 @@ namespace ccfapp
               consensus, view, seqno, error_reason);
           };
 
-        ccf::historical::adapter_v2(
+        ccf::historical::adapter_v3(
           [this, endpoint](
             ccf::endpoints::EndpointContext& endpoint_ctx,
             ccf::historical::StatePtr state) {
@@ -255,7 +255,7 @@ namespace ccfapp
             assert(receipt);
             do_execute_request(endpoint, endpoint_ctx, &tx, tx_id, receipt);
           },
-          context.get_historical_state(),
+          context,
           is_tx_committed)(endpoint_ctx);
       }
       else

src/apps/js_v8/js_v8_base.cpp

@@ -49,13 +49,13 @@ namespace ccfapp
               consensus, view, seqno, error_reason);
           };
 
-        ccf::historical::adapter_v2(
+        ccf::historical::adapter_v3(
           [this, endpoint_def](
             ccf::endpoints::EndpointContext& endpoint_ctx,
             ccf::historical::StatePtr state) {
             do_execute_request(endpoint_def, endpoint_ctx, state);
           },
-          context.get_historical_state(),
+          context,
           is_tx_committed)(endpoint_ctx);
       }
       else

src/crypto/certs.h

@@ -76,4 +76,23 @@ namespace crypto
       issuer_private_key,
       issuer_cert);
   }
+
+  static Pem create_endorsed_cert(
+    const Pem& public_key,
+    const std::string& subject_name,
+    const std::vector<SubjectAltName>& subject_alt_names,
+    const std::string& valid_from,
+    size_t validity_period_days,
+    const Pem& issuer_private_key,
+    const Pem& issuer_cert,
+    bool ca = false)
+  {
+    auto issuer_key_pair = make_key_pair(issuer_private_key);
+    auto csr =
+      issuer_key_pair->create_csr(subject_name, subject_alt_names, public_key);
+    auto valid_to =
+      compute_cert_valid_to_string(valid_from, validity_period_days);
+    return issuer_key_pair->sign_csr(
+      issuer_cert, csr, valid_from, valid_to, ca, KeyPair::Signer::ISSUER);
+  }
 }