Move PeerDAS type level config to ChainSpec. (WIP)

beacon_node/beacon_chain/benches/benches.rs

@@ -34,6 +34,7 @@ fn create_test_block_and_blobs<E: EthSpec>(
 
 fn all_benches(c: &mut Criterion) {
     type E = MainnetEthSpec;
+    let spec = E::default_spec();
 
     let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP_BYTES)
         .map_err(|e| format!("Unable to read trusted setup file: {}", e))
@@ -42,11 +43,11 @@ fn all_benches(c: &mut Criterion) {
 
     for blob_count in [1, 2, 3, 6] {
         let kzg = kzg.clone();
-        let spec = E::default_spec();
         let (signed_block, blob_sidecars) = create_test_block_and_blobs::<E>(blob_count, &spec);
 
         let column_sidecars =
-            DataColumnSidecar::build_sidecars(&blob_sidecars, &signed_block, &kzg.clone()).unwrap();
+            DataColumnSidecar::build_sidecars(&blob_sidecars, &signed_block, &kzg.clone(), &spec)
+                .unwrap();
 
         c.bench(
             &format!("reconstruct_{}", blob_count),

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -125,7 +125,7 @@ use task_executor::{ShutdownReason, TaskExecutor};
 use tokio_stream::Stream;
 use tree_hash::TreeHash;
 use types::blob_sidecar::FixedBlobSidecarList;
-use types::data_column_sidecar::{ColumnIndex, DataColumnIdentifier, DataColumnSidecarList};
+use types::data_column_sidecar::{ColumnIndex, DataColumnIdentifier};
 use types::payload::BlockProductionVersion;
 use types::*;
 
@@ -1324,7 +1324,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     ) -> Result<DataColumnSidecarList<T::EthSpec>, Error> {
         match self.store.get_data_columns(block_root)? {
             Some(data_columns) => Ok(data_columns),
-            None => Ok(DataColumnSidecarList::default()),
+            None => Ok(RuntimeVariableList::empty(self.spec.number_of_columns)),
         }
     }
 

beacon_node/beacon_chain/src/block_verification.rs

@@ -101,7 +101,7 @@ use types::data_column_sidecar::DataColumnSidecarError;
 use types::{
     BeaconBlockRef, BeaconState, BeaconStateError, BlobsList, ChainSpec, DataColumnSidecar,
     DataColumnSubnetId, Epoch, EthSpec, ExecutionBlockHash, FullPayload, Hash256, InconsistentFork,
-    KzgProofs, PublicKey, PublicKeyBytes, RelativeEpoch, SignedBeaconBlock,
+    KzgProofs, PublicKey, PublicKeyBytes, RelativeEpoch, RuntimeVariableList, SignedBeaconBlock,
     SignedBeaconBlockHeader, Slot,
 };
 use types::{BlobSidecar, ExecPayload};
@@ -801,18 +801,22 @@ fn build_gossip_verified_data_columns<T: BeaconChainTypes>(
                 ))?;
 
             let timer = metrics::start_timer(&metrics::DATA_COLUMN_SIDECAR_COMPUTATION);
-            let sidecars = DataColumnSidecar::build_sidecars(&blobs, block, kzg)?;
+            let sidecars = DataColumnSidecar::build_sidecars(&blobs, block, kzg, &chain.spec)?;
             drop(timer);
             let mut gossip_verified_data_columns = vec![];
             for sidecar in sidecars {
-                let subnet =
-                    DataColumnSubnetId::from_column_index::<T::EthSpec>(sidecar.index as usize);
+                let subnet = DataColumnSubnetId::from_column_index::<T::EthSpec>(
+                    sidecar.index as usize,
+                    &chain.spec,
+                );
                 let column = GossipVerifiedDataColumn::new(sidecar, subnet.into(), chain)?;
                 gossip_verified_data_columns.push(column);
             }
-            let gossip_verified_data_columns =
-                GossipVerifiedDataColumnList::new(gossip_verified_data_columns)
-                    .map_err(DataColumnSidecarError::SszError)?;
+            let gossip_verified_data_columns = RuntimeVariableList::new(
+                gossip_verified_data_columns,
+                chain.spec.number_of_columns,
+            )
+            .map_err(DataColumnSidecarError::SszError)?;
             Ok::<_, BlockContentsError<T::EthSpec>>(gossip_verified_data_columns)
         })
         .transpose()

beacon_node/beacon_chain/src/block_verification_types.rs

@@ -15,8 +15,8 @@ use std::sync::Arc;
 use types::blob_sidecar::{self, BlobIdentifier, FixedBlobSidecarList};
 use types::data_column_sidecar::{self};
 use types::{
-    BeaconBlockRef, BeaconState, BlindedPayload, BlobSidecarList, Epoch, EthSpec, Hash256,
-    SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
+    BeaconBlockRef, BeaconState, BlindedPayload, BlobSidecarList, ChainSpec, Epoch, EthSpec,
+    Hash256, RuntimeVariableList, SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
 };
 
 /// A block that has been received over RPC. It has 2 internal variants:
@@ -158,6 +158,7 @@ impl<E: EthSpec> RpcBlock<E> {
         block_root: Option<Hash256>,
         block: Arc<SignedBeaconBlock<E>>,
         custody_columns: Vec<CustodyDataColumn<E>>,
+        spec: &ChainSpec,
     ) -> Result<Self, AvailabilityCheckError> {
         let block_root = block_root.unwrap_or_else(|| get_block_root(&block));
 
@@ -167,7 +168,10 @@ impl<E: EthSpec> RpcBlock<E> {
         }
         // Treat empty data column lists as if they are missing.
         let inner = if !custody_columns.is_empty() {
-            RpcBlockInner::BlockAndCustodyColumns(block, VariableList::new(custody_columns)?)
+            RpcBlockInner::BlockAndCustodyColumns(
+                block,
+                RuntimeVariableList::new(custody_columns, spec.number_of_columns)?,
+            )
         } else {
             RpcBlockInner::Block(block)
         };
@@ -592,6 +596,7 @@ impl<E: EthSpec> AsBlock<E> for AvailableBlock<E> {
     }
 
     fn into_rpc_block(self) -> RpcBlock<E> {
+        let number_of_columns = self.spec.number_of_columns;
         let (block_root, block, blobs_opt, data_columns_opt) = self.deconstruct();
         // Circumvent the constructor here, because an Available block will have already had
         // consistency checks performed.
@@ -600,7 +605,7 @@ impl<E: EthSpec> AsBlock<E> for AvailableBlock<E> {
             (Some(blobs), _) => RpcBlockInner::BlockAndBlobs(block, blobs),
             (_, Some(data_columns)) => RpcBlockInner::BlockAndCustodyColumns(
                 block,
-                VariableList::new(
+                RuntimeVariableList::new(
                     data_columns
                         .into_iter()
                         // TODO(das): This is an ugly hack that should be removed. After updating
@@ -609,6 +614,7 @@ impl<E: EthSpec> AsBlock<E> for AvailableBlock<E> {
                         // columns.
                         .map(|d| CustodyDataColumn::from_asserted_custody(d))
                         .collect(),
+                    number_of_columns,
                 )
                 .expect("data column list is within bounds"),
             ),

beacon_node/beacon_chain/src/builder.rs

@@ -11,6 +11,7 @@ use crate::graffiti_calculator::{GraffitiCalculator, GraffitiOrigin};
 use crate::head_tracker::HeadTracker;
 use crate::light_client_server_cache::LightClientServerCache;
 use crate::migrate::{BackgroundMigrator, MigratorConfig};
+use crate::observed_data_sidecars::ObservedDataSidecars;
 use crate::persisted_beacon_chain::PersistedBeaconChain;
 use crate::shuffling_cache::{BlockShufflingIds, ShufflingCache};
 use crate::timeout_rw_lock::TimeoutRwLock;
@@ -936,8 +937,8 @@ where
             observed_sync_aggregators: <_>::default(),
             // TODO: allow for persisting and loading the pool from disk.
             observed_block_producers: <_>::default(),
-            observed_column_sidecars: <_>::default(),
-            observed_blob_sidecars: <_>::default(),
+            observed_column_sidecars: RwLock::new(ObservedDataSidecars::new(self.spec.clone())),
+            observed_blob_sidecars: RwLock::new(ObservedDataSidecars::new(self.spec.clone())),
             observed_slashable: <_>::default(),
             observed_voluntary_exits: <_>::default(),
             observed_proposer_slashings: <_>::default(),

beacon_node/beacon_chain/src/data_availability_checker.rs

@@ -7,7 +7,6 @@ use crate::{BeaconChain, BeaconChainTypes, BeaconStore};
 use kzg::Kzg;
 use slog::{debug, error, o, Logger};
 use slot_clock::SlotClock;
-use ssz_types::VariableList;
 use std::fmt;
 use std::fmt::Debug;
 use std::num::NonZeroUsize;
@@ -16,7 +15,8 @@ use std::time::Duration;
 use task_executor::TaskExecutor;
 use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList};
 use types::{
-    BlobSidecarList, ChainSpec, DataColumnSidecar, Epoch, EthSpec, Hash256, SignedBeaconBlock,
+    BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarList, Epoch, EthSpec, Hash256,
+    RuntimeVariableList, SignedBeaconBlock,
 };
 
 mod error;
@@ -28,7 +28,7 @@ use crate::data_column_verification::{
     KzgVerifiedCustodyDataColumn,
 };
 pub use error::{Error as AvailabilityCheckError, ErrorCategory as AvailabilityCheckErrorCategory};
-use types::data_column_sidecar::{DataColumnIdentifier, DataColumnSidecarList};
+use types::data_column_sidecar::DataColumnIdentifier;
 use types::non_zero_usize::new_non_zero_usize;
 
 pub use self::overflow_lru_cache::DataColumnsToPublish;
@@ -51,7 +51,7 @@ pub struct DataAvailabilityChecker<T: BeaconChainTypes> {
     availability_cache: Arc<OverflowLRUCache<T>>,
     slot_clock: T::SlotClock,
     kzg: Option<Arc<Kzg>>,
-    spec: ChainSpec,
+    spec: Arc<ChainSpec>,
 }
 
 /// This type is returned after adding a block / blob to the `DataAvailabilityChecker`.
@@ -84,14 +84,15 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
         log: &Logger,
         spec: ChainSpec,
     ) -> Result<Self, AvailabilityCheckError> {
+        let spec = Arc::new(spec);
         let custody_subnet_count = if import_all_data_columns {
-            T::EthSpec::data_column_subnet_count()
+            spec.data_column_sidecar_subnet_count as usize
         } else {
-            T::EthSpec::custody_requirement()
+            spec.custody_requirement as usize
         };
 
         let custody_column_count =
-            custody_subnet_count.saturating_mul(T::EthSpec::data_columns_per_subnet());
+            custody_subnet_count.saturating_mul(spec.data_columns_per_subnet());
         let overflow_cache = OverflowLRUCache::new(
             OVERFLOW_LRU_CAPACITY,
             store,
@@ -289,6 +290,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
                     blobs,
                     blobs_available_timestamp: None,
                     data_columns: None,
+                    spec: self.spec.clone(),
                 }))
             } else {
                 Ok(MaybeAvailableBlock::AvailabilityPending { block_root, block })
@@ -314,14 +316,16 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
                     blobs_available_timestamp: None,
                     // TODO(das): update store type to prevent this conversion
                     data_columns: Some(
-                        VariableList::new(
+                        RuntimeVariableList::new(
                             data_column_list
                                 .into_iter()
                                 .map(|d| d.clone_arc())
                                 .collect(),
+                            self.spec.number_of_columns,
                         )
                         .expect("data column list is within bounds"),
                     ),
+                    spec: self.spec.clone(),
                 }))
             } else {
                 Ok(MaybeAvailableBlock::AvailabilityPending { block_root, block })
@@ -334,6 +338,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
             blobs: None,
             blobs_available_timestamp: None,
             data_columns: None,
+            spec: self.spec.clone(),
         }))
     }
 
@@ -366,15 +371,16 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
             verify_kzg_for_blob_list(all_blobs.iter(), kzg)?;
         }
 
-        let all_data_columns: DataColumnSidecarList<T::EthSpec> = blocks
+        let all_data_columns = blocks
             .iter()
             .filter(|block| self.data_columns_required_for_block(block.as_block()))
             // this clone is cheap as it's cloning an Arc
             .filter_map(|block| block.custody_columns().cloned())
             .flatten()
             .map(CustodyDataColumn::into_inner)
-            .collect::<Vec<_>>()
-            .into();
+            .collect::<Vec<_>>();
+        let all_data_columns =
+            RuntimeVariableList::from_vec(all_data_columns, self.spec.number_of_columns);
 
         // verify kzg for all data columns at once
         if !all_data_columns.is_empty() {
@@ -396,6 +402,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
                         blobs,
                         blobs_available_timestamp: None,
                         data_columns: None,
+                        spec: self.spec.clone(),
                     })
                 } else {
                     MaybeAvailableBlock::AvailabilityPending { block_root, block }
@@ -409,11 +416,13 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
                         blobs_available_timestamp: None,
                         // TODO(das): update store type to prevent this conversion
                         data_columns: data_columns.map(|data_columns| {
-                            VariableList::new(
+                            RuntimeVariableList::new(
                                 data_columns.into_iter().map(|d| d.into_inner()).collect(),
+                                self.spec.number_of_columns,
                             )
                             .expect("data column list is within bounds")
                         }),
+                        spec: self.spec.clone(),
                     })
                 } else {
                     MaybeAvailableBlock::AvailabilityPending { block_root, block }
@@ -425,6 +434,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
                     blobs: None,
                     blobs_available_timestamp: None,
                     data_columns: None,
+                    spec: self.spec.clone(),
                 })
             };
 
@@ -601,6 +611,7 @@ pub struct AvailableBlock<E: EthSpec> {
     /// Timestamp at which this block first became available (UNIX timestamp, time since 1970).
     blobs_available_timestamp: Option<Duration>,
     data_columns: Option<DataColumnSidecarList<E>>,
+    pub spec: Arc<ChainSpec>,
 }
 
 impl<E: EthSpec> AvailableBlock<E> {
@@ -609,13 +620,15 @@ impl<E: EthSpec> AvailableBlock<E> {
         block: Arc<SignedBeaconBlock<E>>,
         blobs: Option<BlobSidecarList<E>>,
         data_columns: Option<DataColumnSidecarList<E>>,
+        spec: Arc<ChainSpec>,
     ) -> Self {
         Self {
             block_root,
             block,
             blobs,
             blobs_available_timestamp: None,
             data_columns,
+            spec,
         }
     }
 
@@ -654,6 +667,7 @@ impl<E: EthSpec> AvailableBlock<E> {
             blobs,
             blobs_available_timestamp: _,
             data_columns,
+            ..
         } = self;
         (block_root, block, blobs, data_columns)
     }