Add blob sidecar inclusion proof

presets/mainnet/deneb.yaml

@@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096
 # `uint64(6)`
 MAX_BLOBS_PER_BLOCK: 6
+# `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17
+KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17

presets/minimal/deneb.yaml

@@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 16
 # `uint64(6)`
 MAX_BLOBS_PER_BLOCK: 6
+# [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9
+KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9

pysetup/helpers.py

@@ -68,8 +68,7 @@ def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str
  if k in [
  "ceillog2",
  "floorlog2",
- "compute_merkle_proof_for_block_body",
- "compute_merkle_proof_for_state",
+ "compute_merkle_proof",
  ]:
  del spec_object.functions[k]
 
@@ -111,8 +110,9 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
  return out
 
  # Merge all constant objects
- hardcoded_ssz_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()},  builders, {})
+ hardcoded_ssz_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()}, builders, {})
  hardcoded_custom_type_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_custom_type_dep_constants(spec_object)}, builders, {})
+ hardcoded_func_dep_presets = reduce(lambda obj, builder: {**obj, **builder.hardcoded_func_dep_presets(spec_object)}, builders, {})
  # Concatenate all strings
  imports = reduce(lambda txt, builder: (txt + "\n\n" + builder.imports(preset_name) ).strip("\n"), builders, "")
  preparations = reduce(lambda txt, builder: (txt + "\n\n" + builder.preparations() ).strip("\n"), builders, "")
@@ -126,6 +126,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
  ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
  ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
  custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_custom_type_dep_constants[x]), hardcoded_custom_type_dep_constants))
+ func_dep_presets_verification = '\n'.join(map(lambda x: 'assert %s == %s # noqa: E501' % (x, spec_object.func_dep_presets[x]), hardcoded_func_dep_presets))
  spec_strs = [
  imports,
  preparations,
@@ -147,6 +148,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
  # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are
  # as same as the spec definition.
  ssz_dep_constants_verification,
+ func_dep_presets_verification,
  ]
  return "\n\n\n".join([str.strip("\n") for str in spec_strs if str]) + "\n"
 
@@ -223,6 +225,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
  preset_vars = combine_dicts(spec0.preset_vars, spec1.preset_vars)
  config_vars = combine_dicts(spec0.config_vars, spec1.config_vars)
  ssz_dep_constants = combine_dicts(spec0.ssz_dep_constants, spec1.ssz_dep_constants)
+ func_dep_presets = combine_dicts(spec0.func_dep_presets, spec1.func_dep_presets)
  ssz_objects = combine_ssz_objects(spec0.ssz_objects, spec1.ssz_objects, custom_types)
  dataclasses = combine_dicts(spec0.dataclasses, spec1.dataclasses)
  return SpecObject(
@@ -233,6 +236,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
  preset_vars=preset_vars,
  config_vars=config_vars,
  ssz_dep_constants=ssz_dep_constants,
+ func_dep_presets=func_dep_presets,
  ssz_objects=ssz_objects,
  dataclasses=dataclasses,
  )

pysetup/spec_builders/altair.py

@@ -27,16 +27,16 @@ def preparations(cls):
  @classmethod
  def sundry_functions(cls) -> str:
  return '''
-def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariableName]]) -> GeneralizedIndex:
+def get_generalized_index(ssz_class: Any, *path: PyUnion[int, SSZVariableName]) -> GeneralizedIndex:
  ssz_path = Path(ssz_class)
  for item in path:
  ssz_path = ssz_path / item
  return GeneralizedIndex(ssz_path.gindex())
 
 
-def compute_merkle_proof_for_state(state: BeaconState,
-  index: GeneralizedIndex) -> Sequence[Bytes32]:
- return build_proof(state.get_backing(), index)'''
+def compute_merkle_proof(object: SSZObject,
+ index: GeneralizedIndex) -> Sequence[Bytes32]:
+ return build_proof(object.get_backing(), index)'''
 
 
  @classmethod

pysetup/spec_builders/base.py

@@ -47,6 +47,10 @@ def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: #
  """
  return {}
 
+ @classmethod
+ def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
+ return {}
+
  @classmethod
  def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]:
  return functions

pysetup/spec_builders/capella.py

@@ -13,15 +13,6 @@ def imports(cls, preset_name: str):
 from eth2spec.bellatrix import {preset_name} as bellatrix
 '''
 
-
- @classmethod
- def sundry_functions(cls) -> str:
- return '''
-def compute_merkle_proof_for_block_body(body: BeaconBlockBody,
- index: GeneralizedIndex) -> Sequence[Bytes32]:
- return build_proof(body.get_backing(), index)'''
-
-
  @classmethod
  def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
  return {

pysetup/spec_builders/deneb.py

@@ -1,3 +1,4 @@
+from typing import Dict
 from .base import BaseSpecBuilder
 from ..constants import DENEB
 
@@ -23,7 +24,8 @@ def sundry_functions(cls) -> str:
  return '''
 def retrieve_blobs_and_proofs(beacon_block_root: Root) -> Tuple[Sequence[Blob], Sequence[KZGProof]]:
  # pylint: disable=unused-argument
- return [], []'''
+ return [], []
+'''
 
  @classmethod
  def execution_engine_cls(cls) -> str:
@@ -63,9 +65,16 @@ def verify_and_notify_new_payload(self: ExecutionEngine,
 
 
  @classmethod
- def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
+ def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
  return {
  'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value,
  'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
  'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
+ 'MAX_BLOB_COMMITMENTS_PER_BLOCK': spec_object.preset_vars['MAX_BLOB_COMMITMENTS_PER_BLOCK'].value,
+ }
+
+ @classmethod
+ def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
+ return {
+ 'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENT_INCLUSION_PROOF_DEPTH'].value,
  }

pysetup/typing.py

@@ -22,6 +22,7 @@ class SpecObject(NamedTuple):
  preset_vars: Dict[str, VariableDefinition]
  config_vars: Dict[str, VariableDefinition]
  ssz_dep_constants: Dict[str, str] # the constants that depend on ssz_objects
+ func_dep_presets: Dict[str, str] # the constants that depend on functions
  ssz_objects: Dict[str, str]
  dataclasses: Dict[str, str]
 

setup.py

@@ -162,6 +162,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
  preset_vars: Dict[str, VariableDefinition] = {}
  config_vars: Dict[str, VariableDefinition] = {}
  ssz_dep_constants: Dict[str, str] = {}
+ func_dep_presets: Dict[str, str] = {}
  ssz_objects: Dict[str, str] = {}
  dataclasses: Dict[str, str] = {}
  custom_types: Dict[str, str] = {}
@@ -214,6 +215,16 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
 
  value_cell = cells[1]
  value = value_cell.children[0].children
+
+ description = None
+ if len(cells) >= 3:
+ description_cell = cells[2]
+ if len(description_cell.children) > 0:
+ description = description_cell.children[0].children
+ if isinstance(description, list):
+ # marko parses `**X**` as a list containing a X
+ description = description[0].children
+
  if isinstance(value, list):
  # marko parses `**X**` as a list containing a X
  value = value[0].children
@@ -228,6 +239,9 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
  ssz_dep_constants[name] = value
  continue
 
+ if description is not None and description.startswith("<!-- predefined -->"):
+ func_dep_presets[name] = value
+
  value_def = _parse_value(name, value)
  if name in preset:
  preset_vars[name] = VariableDefinition(value_def.type_name, preset[name], value_def.comment, None)
@@ -256,6 +270,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
  preset_vars=preset_vars,
  config_vars=config_vars,
  ssz_dep_constants=ssz_dep_constants,
+ func_dep_presets=func_dep_presets,
  ssz_objects=ssz_objects,
  dataclasses=dataclasses,
  )

specs/altair/light-client/full-node.md

@@ -10,7 +10,7 @@
 
 - [Introduction](#introduction)
 - [Helper functions](#helper-functions)
- - [`compute_merkle_proof_for_state`](#compute_merkle_proof_for_state)
+ - [`compute_merkle_proof`](#compute_merkle_proof)
  - [`block_to_light_client_header`](#block_to_light_client_header)
 - [Deriving light client data](#deriving-light-client-data)
  - [`create_light_client_bootstrap`](#create_light_client_bootstrap)
@@ -27,11 +27,13 @@ This document provides helper functions to enable full nodes to serve light clie
 
 ## Helper functions
 
-### `compute_merkle_proof_for_state`
+### `compute_merkle_proof`
+
+This function return the Merkle proof of the given SSZ object `object` at generalized index `index`.
 
 ```python
-def compute_merkle_proof_for_state(state: BeaconState,
-  index: GeneralizedIndex) -> Sequence[Bytes32]:
+def compute_merkle_proof(object: SSZObject,
+ index: GeneralizedIndex) -> Sequence[Bytes32]:
  ...
 ```
 
@@ -73,7 +75,7 @@ def create_light_client_bootstrap(state: BeaconState,
  return LightClientBootstrap(
  header=block_to_light_client_header(block),
  current_sync_committee=state.current_sync_committee,
- current_sync_committee_branch=compute_merkle_proof_for_state(state, CURRENT_SYNC_COMMITTEE_INDEX),
+ current_sync_committee_branch=compute_merkle_proof(state, CURRENT_SYNC_COMMITTEE_INDEX),
  )
 ```
 
@@ -120,8 +122,7 @@ def create_light_client_update(state: BeaconState,
  # `next_sync_committee` is only useful if the message is signed by the current sync committee
  if update_attested_period == update_signature_period:
  update.next_sync_committee = attested_state.next_sync_committee
- update.next_sync_committee_branch = compute_merkle_proof_for_state(
- attested_state, NEXT_SYNC_COMMITTEE_INDEX)
+ update.next_sync_committee_branch = compute_merkle_proof(attested_state, NEXT_SYNC_COMMITTEE_INDEX)
 
  # Indicate finality whenever possible
  if finalized_block is not None:
@@ -130,8 +131,7 @@ def create_light_client_update(state: BeaconState,
  assert hash_tree_root(update.finalized_header.beacon) == attested_state.finalized_checkpoint.root
  else:
  assert attested_state.finalized_checkpoint.root == Bytes32()
- update.finality_branch = compute_merkle_proof_for_state(
- attested_state, FINALIZED_ROOT_INDEX)
+ update.finality_branch = compute_merkle_proof(attested_state, FINALIZED_ROOT_INDEX)
 
  update.sync_aggregate = block.message.body.sync_aggregate
  update.signature_slot = block.message.slot

specs/capella/light-client/full-node.md

@@ -10,7 +10,6 @@
 
 - [Introduction](#introduction)
 - [Helper functions](#helper-functions)
- - [`compute_merkle_proof_for_block_body`](#compute_merkle_proof_for_block_body)
  - [Modified `block_to_light_client_header`](#modified-block_to_light_client_header)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -22,14 +21,6 @@ This upgrade adds information about the execution payload to light client data a
 
 ## Helper functions
 
-### `compute_merkle_proof_for_block_body`
-
-```python
-def compute_merkle_proof_for_block_body(body: BeaconBlockBody,
- index: GeneralizedIndex) -> Sequence[Bytes32]:
- ...
-```
-
 ### Modified `block_to_light_client_header`
 
 ```python
@@ -55,7 +46,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader:
  transactions_root=hash_tree_root(payload.transactions),
  withdrawals_root=hash_tree_root(payload.withdrawals),
  )
- execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX)
+ execution_branch = compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_INDEX)
  else:
  # Note that during fork transitions, `finalized_header` may still point to earlier forks.
  # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`),

specs/deneb/light-client/full-node.md

@@ -52,7 +52,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader:
  execution_header.blob_gas_used = payload.blob_gas_used
  execution_header.excess_blob_gas = payload.excess_blob_gas
 
- execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX)
+ execution_branch = compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_INDEX)
  else:
  # Note that during fork transitions, `finalized_header` may still point to earlier forks.
  # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`),