Skip to content

Commit

Permalink
Create DataColumnSidecars on block production and publish it (Consens…
Browse files Browse the repository at this point in the history
  • Loading branch information
zilm13 authored May 1, 2024
1 parent b3e8079 commit 2a12f22
Show file tree
Hide file tree
Showing 11 changed files with 196 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import tech.pegasys.teku.infrastructure.async.SafeFuture;
import tech.pegasys.teku.infrastructure.unsigned.UInt64;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
import tech.pegasys.teku.spec.datastructures.blobs.versions.eip7594.DataColumnSidecar;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
import tech.pegasys.teku.spec.datastructures.metadata.BlockContainerAndMetaData;
Expand All @@ -43,4 +44,6 @@ SafeFuture<SignedBeaconBlock> unblindSignedBlockIfBlinded(

List<BlobSidecar> createBlobSidecars(
SignedBlockContainer blockContainer, BlockPublishingPerformance blockPublishingPerformance);

List<DataColumnSidecar> createDataColumnSidecars(SignedBlockContainer blockContainer);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright Consensys Software Inc., 2023
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/

package tech.pegasys.teku.validator.coordinator;

import java.util.List;
import tech.pegasys.teku.kzg.KZG;
import tech.pegasys.teku.spec.Spec;
import tech.pegasys.teku.spec.datastructures.blobs.versions.eip7594.DataColumnSidecar;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
import tech.pegasys.teku.spec.logic.versions.eip7594.helpers.MiscHelpersEip7594;

public class BlockFactoryEip7594 extends BlockFactoryDeneb {
private final KZG kzg;

public BlockFactoryEip7594(
final Spec spec, final BlockOperationSelectorFactory operationSelector, final KZG kzg) {
super(spec, operationSelector);
this.kzg = kzg;
}

@Override
public List<DataColumnSidecar> createDataColumnSidecars(
final SignedBlockContainer blockContainer) {
final MiscHelpersEip7594 miscHelpersEip7594 =
MiscHelpersEip7594.required(spec.atSlot(blockContainer.getSlot()).miscHelpers());
return miscHelpersEip7594.constructDataColumnSidecars(
blockContainer.getSignedBlock(),
blockContainer.getBlobs().orElseThrow().asList(),
blockContainer.getKzgProofs().orElseThrow(),
kzg);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import tech.pegasys.teku.infrastructure.unsigned.UInt64;
import tech.pegasys.teku.spec.Spec;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
import tech.pegasys.teku.spec.datastructures.blobs.versions.eip7594.DataColumnSidecar;
import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockAndState;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
Expand Down Expand Up @@ -111,4 +112,10 @@ public List<BlobSidecar> createBlobSidecars(
final BlockPublishingPerformance blockPublishingPerformance) {
return Collections.emptyList();
}

@Override
public List<DataColumnSidecar> createDataColumnSidecars(
final SignedBlockContainer blockContainer) {
return Collections.emptyList();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,11 @@
import tech.pegasys.teku.ethereum.performance.trackers.BlockPublishingPerformance;
import tech.pegasys.teku.infrastructure.async.SafeFuture;
import tech.pegasys.teku.infrastructure.unsigned.UInt64;
import tech.pegasys.teku.kzg.KZG;
import tech.pegasys.teku.spec.Spec;
import tech.pegasys.teku.spec.SpecMilestone;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
import tech.pegasys.teku.spec.datastructures.blobs.versions.eip7594.DataColumnSidecar;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
import tech.pegasys.teku.spec.datastructures.metadata.BlockContainerAndMetaData;
Expand All @@ -40,20 +42,24 @@ public class MilestoneBasedBlockFactory implements BlockFactory {
private final Spec spec;

public MilestoneBasedBlockFactory(
final Spec spec, final BlockOperationSelectorFactory operationSelector) {
final Spec spec, final BlockOperationSelectorFactory operationSelector, final KZG kzg) {
this.spec = spec;
final BlockFactoryPhase0 blockFactoryPhase0 = new BlockFactoryPhase0(spec, operationSelector);

// Not needed for all milestones
final Supplier<BlockFactoryDeneb> blockFactoryDenebSupplier =
Suppliers.memoize(() -> new BlockFactoryDeneb(spec, operationSelector));
final Supplier<BlockFactoryEip7594> blockFactoryEip7594Supplier =
Suppliers.memoize(() -> new BlockFactoryEip7594(spec, operationSelector, kzg));

// Populate forks factories
spec.getEnabledMilestones()
.forEach(
forkAndSpecMilestone -> {
final SpecMilestone milestone = forkAndSpecMilestone.getSpecMilestone();
if (milestone.isGreaterThanOrEqualTo(SpecMilestone.DENEB)) {
if (milestone.isGreaterThanOrEqualTo(SpecMilestone.EIP7594)) {
registeredFactories.put(milestone, blockFactoryEip7594Supplier.get());
} else if (milestone.equals(SpecMilestone.DENEB)) {
registeredFactories.put(milestone, blockFactoryDenebSupplier.get());
} else {
registeredFactories.put(milestone, blockFactoryPhase0);
Expand Down Expand Up @@ -103,6 +109,13 @@ public List<BlobSidecar> createBlobSidecars(
.createBlobSidecars(blockContainer, blockPublishingPerformance);
}

@Override
public List<DataColumnSidecar> createDataColumnSidecars(
final SignedBlockContainer blockContainer) {
final SpecMilestone milestone = getMilestone(blockContainer.getSlot());
return registeredFactories.get(milestone).createDataColumnSidecars(blockContainer);
}

private SpecMilestone getMilestone(final UInt64 slot) {
return spec.atSlot(slot).getMilestone();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import tech.pegasys.teku.infrastructure.unsigned.UInt64;
import tech.pegasys.teku.networking.eth2.gossip.BlobSidecarGossipChannel;
import tech.pegasys.teku.networking.eth2.gossip.BlockGossipChannel;
import tech.pegasys.teku.networking.eth2.gossip.DataColumnSidecarGossipChannel;
import tech.pegasys.teku.networking.eth2.gossip.subnets.AttestationTopicSubscriber;
import tech.pegasys.teku.networking.eth2.gossip.subnets.SyncCommitteeSubscriptionManager;
import tech.pegasys.teku.spec.Spec;
Expand Down Expand Up @@ -149,6 +150,7 @@ public ValidatorApiHandler(
final BlockGossipChannel blockGossipChannel,
final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool,
final BlobSidecarGossipChannel blobSidecarGossipChannel,
final DataColumnSidecarGossipChannel dataColumnSidecarGossipChannel,
final AggregatingAttestationPool attestationPool,
final AttestationManager attestationManager,
final AttestationTopicSubscriber attestationTopicSubscriber,
Expand Down Expand Up @@ -190,6 +192,7 @@ public ValidatorApiHandler(
blockGossipChannel,
blockBlobSidecarsTrackersPool,
blobSidecarGossipChannel,
dataColumnSidecarGossipChannel,
performanceTracker,
dutyMetrics);
this.attesterDutiesGenerator = new AttesterDutiesGenerator(spec);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Copyright Consensys Software Inc., 2022
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/

package tech.pegasys.teku.validator.coordinator.publisher;

import java.util.List;
import tech.pegasys.teku.ethereum.performance.trackers.BlockPublishingPerformance;
import tech.pegasys.teku.infrastructure.async.SafeFuture;
import tech.pegasys.teku.networking.eth2.gossip.BlockGossipChannel;
import tech.pegasys.teku.networking.eth2.gossip.DataColumnSidecarGossipChannel;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
import tech.pegasys.teku.spec.datastructures.blobs.versions.eip7594.DataColumnSidecar;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
import tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel;
import tech.pegasys.teku.statetransition.blobs.BlockBlobSidecarsTrackersPool;
import tech.pegasys.teku.statetransition.block.BlockImportChannel;
import tech.pegasys.teku.statetransition.block.BlockImportChannel.BlockImportAndBroadcastValidationResults;
import tech.pegasys.teku.validator.coordinator.BlockFactory;
import tech.pegasys.teku.validator.coordinator.DutyMetrics;
import tech.pegasys.teku.validator.coordinator.performance.PerformanceTracker;

public class BlockPublisherEip7594 extends AbstractBlockPublisher {

private final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool;
private final BlockGossipChannel blockGossipChannel;
private final DataColumnSidecarGossipChannel dataColumnSidecarGossipChannel;

public BlockPublisherEip7594(
final BlockFactory blockFactory,
final BlockImportChannel blockImportChannel,
final BlockGossipChannel blockGossipChannel,
final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool,
final DataColumnSidecarGossipChannel dataColumnSidecarGossipChannel,
final PerformanceTracker performanceTracker,
final DutyMetrics dutyMetrics) {
super(blockFactory, blockImportChannel, performanceTracker, dutyMetrics);
this.blockBlobSidecarsTrackersPool = blockBlobSidecarsTrackersPool;
this.blockGossipChannel = blockGossipChannel;
this.dataColumnSidecarGossipChannel = dataColumnSidecarGossipChannel;
}

@Override
SafeFuture<BlockImportAndBroadcastValidationResults> importBlockAndBlobSidecars(
final SignedBeaconBlock block,
final List<BlobSidecar> blobSidecars,
final BroadcastValidationLevel broadcastValidationLevel,
final BlockPublishingPerformance blockPublishingPerformance) {
// TODO: DataColumnSidecars pool fill up
// provide blobs for the block before importing it
blockBlobSidecarsTrackersPool.onCompletedBlockAndBlobSidecars(block, blobSidecars);
return blockImportChannel
.importBlock(block, broadcastValidationLevel)
.thenPeek(__ -> blockPublishingPerformance.blockImportCompleted());
}

@Override
void publishBlockAndBlobSidecars(
final SignedBeaconBlock block,
final List<BlobSidecar> blobSidecars,
BlockPublishingPerformance blockPublishingPerformance) {
blockGossipChannel.publishBlock(block);
final List<DataColumnSidecar> dataColumnSidecars = blockFactory.createDataColumnSidecars(block);
dataColumnSidecarGossipChannel.publishDataColumnSidecars(dataColumnSidecars);
blockPublishingPerformance.blockAndBlobSidecarsPublishingInitiated();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import tech.pegasys.teku.infrastructure.async.SafeFuture;
import tech.pegasys.teku.networking.eth2.gossip.BlobSidecarGossipChannel;
import tech.pegasys.teku.networking.eth2.gossip.BlockGossipChannel;
import tech.pegasys.teku.networking.eth2.gossip.DataColumnSidecarGossipChannel;
import tech.pegasys.teku.spec.Spec;
import tech.pegasys.teku.spec.SpecMilestone;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
Expand All @@ -44,6 +45,7 @@ public MilestoneBasedBlockPublisher(
final BlockGossipChannel blockGossipChannel,
final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool,
final BlobSidecarGossipChannel blobSidecarGossipChannel,
final DataColumnSidecarGossipChannel dataColumnSidecarGossipChannel,
final PerformanceTracker performanceTracker,
final DutyMetrics dutyMetrics) {
this.spec = spec;
Expand All @@ -63,13 +65,27 @@ public MilestoneBasedBlockPublisher(
blobSidecarGossipChannel,
performanceTracker,
dutyMetrics));
final Supplier<BlockPublisherEip7594> blockAndDataColumnSidecarsPublisherSupplier =
Suppliers.memoize(
() ->
new BlockPublisherEip7594(
blockFactory,
blockImportChannel,
blockGossipChannel,
blockBlobSidecarsTrackersPool,
dataColumnSidecarGossipChannel,
performanceTracker,
dutyMetrics));

// Populate forks publishers
spec.getEnabledMilestones()
.forEach(
forkAndSpecMilestone -> {
final SpecMilestone milestone = forkAndSpecMilestone.getSpecMilestone();
if (milestone.isGreaterThanOrEqualTo(SpecMilestone.DENEB)) {
if (milestone.equals(SpecMilestone.EIP7594)) {
registeredPublishers.put(
milestone, blockAndDataColumnSidecarsPublisherSupplier.get());
} else if (milestone.equals(SpecMilestone.DENEB)) {
registeredPublishers.put(milestone, blockAndBlobSidecarsPublisherSupplier.get());
} else {
registeredPublishers.put(milestone, blockPublisherPhase0);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
import tech.pegasys.teku.kzg.KZGProof;
import tech.pegasys.teku.networking.eth2.gossip.BlobSidecarGossipChannel;
import tech.pegasys.teku.networking.eth2.gossip.BlockGossipChannel;
import tech.pegasys.teku.networking.eth2.gossip.DataColumnSidecarGossipChannel;
import tech.pegasys.teku.networking.eth2.gossip.subnets.AttestationTopicSubscriber;
import tech.pegasys.teku.networking.eth2.gossip.subnets.SyncCommitteeSubscriptionManager;
import tech.pegasys.teku.spec.Spec;
Expand Down Expand Up @@ -150,6 +151,8 @@ class ValidatorApiHandlerTest {
mock(BlockBlobSidecarsTrackersPool.class);
private final BlobSidecarGossipChannel blobSidecarGossipChannel =
mock(BlobSidecarGossipChannel.class);
private final DataColumnSidecarGossipChannel dataColumnSidecarGossipChannel =
mock(DataColumnSidecarGossipChannel.class);
private final DefaultPerformanceTracker performanceTracker =
mock(DefaultPerformanceTracker.class);
private final ChainDataProvider chainDataProvider = mock(ChainDataProvider.class);
Expand Down Expand Up @@ -202,6 +205,7 @@ public void setUp() {
blockGossipChannel,
blockBlobSidecarsTrackersPool,
blobSidecarGossipChannel,
dataColumnSidecarGossipChannel,
attestationPool,
attestationManager,
attestationTopicSubscriptions,
Expand Down Expand Up @@ -456,6 +460,7 @@ void getSyncCommitteeDuties_shouldNotUseEpochPriorToFork() {
blockGossipChannel,
blockBlobSidecarsTrackersPool,
blobSidecarGossipChannel,
dataColumnSidecarGossipChannel,
attestationPool,
attestationManager,
attestationTopicSubscriptions,
Expand Down Expand Up @@ -1302,6 +1307,7 @@ private void setupDeneb() {
blockGossipChannel,
blockBlobSidecarsTrackersPool,
blobSidecarGossipChannel,
dataColumnSidecarGossipChannel,
attestationPool,
attestationManager,
attestationTopicSubscriptions,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
import tech.pegasys.teku.validator.coordinator.performance.PerformanceTracker;

public class AbstractBlockPublisherTest {
private final Spec spec = TestSpecFactory.createMinimalDeneb();
private final Spec spec = TestSpecFactory.createMinimalEip7594();
private final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec);
private final BlockFactory blockFactory = mock(BlockFactory.class);
private final BlockImportChannel blockImportChannel = mock(BlockImportChannel.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

package tech.pegasys.teku.spec.logic.versions.eip7594.helpers;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -167,23 +166,21 @@ public List<DataColumnSidecar> constructDataColumnSidecars(
final DataColumnSidecarSchema dataColumnSidecarSchema =
schemaDefinitions.getDataColumnSidecarSchema();

final List<DataColumnSidecar> dataColumnSidecars = new ArrayList<>();
for (int i = 0; i < extendedMatrix.get(0).size(); ++i) {
final int cellID = i;
final DataColumn dataColumn =
dataColumnSchema.create(extendedMatrix.stream().map(row -> row.get(cellID)).toList());
final DataColumnSidecar dataColumnSidecar =
dataColumnSidecarSchema.create(
UInt64.valueOf(cellID),
dataColumn,
sszKZGCommitments,
sszKZGProofs,
signedBeaconBlock.asHeader(),
kzgCommitmentsInclusionProof);
dataColumnSidecars.add(dataColumnSidecar);
}

return dataColumnSidecars;
return IntStream.range(0, extendedMatrix.get(0).size())
.mapToObj(
cellID -> {
final DataColumn dataColumn =
dataColumnSchema.create(
extendedMatrix.stream().map(row -> row.get(cellID)).toList());
return dataColumnSidecarSchema.create(
UInt64.valueOf(cellID),
dataColumn,
sszKZGCommitments,
sszKZGProofs,
signedBeaconBlock.asHeader(),
kzgCommitmentsInclusionProof);
})
.toList();
}

private List<List<Cell>> computeExtendedMatrix(final List<Blob> blobs, final KZG kzg) {
Expand Down
Loading

0 comments on commit 2a12f22

Please sign in to comment.