Merge branch 'master' into update_prometheus

CHANGELOG.md

@@ -12,7 +12,10 @@ the [releases page](https://github.com/Consensys/teku/releases).
 ### Breaking Changes
 
 ### Additions and Improvements
+
 - Added metadata fields to `/eth/v1/beacon/blob_sidecars/{block_id}` Beacon API response as per https://github.com/ethereum/beacon-APIs/pull/441
 - Added rest api endpoint `/teku/v1/beacon/state/finalized/slot/before/{slot}` to return most recent stored state at or before a specified slot.
+- The validator client will start using the `v2` variant of the beacon node block publishing
+ endpoints. In the cases where the block has been produced in the same beacon node, only equivocation validation will be done instead of the entire gossip validation.
 
 ### Bug Fixes

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/RemoteValidatorCompatibilityAcceptanceTest.java

@@ -29,7 +29,7 @@ public class RemoteValidatorCompatibilityAcceptanceTest extends AcceptanceTestBa
 
  @Test
  void shouldRunUpdatedValidatorAgainstOldBeaconNode() throws Exception {
- verifyCompatibility(TekuDockerVersion.V23_9_0, TekuDockerVersion.LOCAL_BUILD);
+ verifyCompatibility(TekuDockerVersion.V24_2_0, TekuDockerVersion.LOCAL_BUILD);
  }
 
  @Test

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuDockerVersion.java

@@ -16,7 +16,7 @@
 public enum TekuDockerVersion {
  LOCAL_BUILD("develop"),
  LAST_RELEASE("latest"),
- V23_9_0("23.9.0");
+ V24_2_0("24.2.0");
 
  private final String version;
 

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandler.java

@@ -21,6 +21,8 @@
 import static tech.pegasys.teku.infrastructure.metrics.Validator.ValidatorDutyMetricUtils.startTimer;
 import static tech.pegasys.teku.infrastructure.metrics.Validator.ValidatorDutyMetricsSteps.CREATE;
 import static tech.pegasys.teku.spec.config.SpecConfig.GENESIS_SLOT;
+import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.EQUIVOCATION;
+import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.GOSSIP;
 
 import com.google.common.annotations.VisibleForTesting;
 import it.unimi.dsi.fastutil.ints.IntCollection;
@@ -32,6 +34,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import org.apache.logging.log4j.LogManager;
@@ -57,6 +60,7 @@
 import tech.pegasys.teku.ethereum.performance.trackers.BlockProductionPerformance;
 import tech.pegasys.teku.ethereum.performance.trackers.BlockPublishingPerformance;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
+import tech.pegasys.teku.infrastructure.collections.LimitedMap;
 import tech.pegasys.teku.infrastructure.ssz.SszList;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.networking.eth2.gossip.BlobSidecarGossipChannel;
@@ -116,6 +120,9 @@ public class ValidatorApiHandler implements ValidatorApiChannel {
  */
  private static final int DUTY_EPOCH_TOLERANCE = 1;
 
+ private final Map<UInt64, Bytes32> createdBlockRootsBySlotCache =
+ LimitedMap.createSynchronizedLRU(2);
+
  private final BlockProductionAndPublishingPerformanceFactory
  blockProductionAndPublishingPerformanceFactory;
  private final ChainDataProvider chainDataProvider;
@@ -376,7 +383,12 @@ private SafeFuture<Optional<BlockContainerAndMetaData>> createBlock(
  requestedBlinded,
  requestedBuilderBoostFactor,
  blockProductionPerformance)
- .thenApply(Optional::of);
+ .thenApply(
+ block -> {
+ final Bytes32 blockRoot = block.blockContainer().getBlock().getRoot();
+ createdBlockRootsBySlotCache.put(slot, blockRoot);
+ return Optional.of(block);
+ });
  }
 
  @Override
@@ -636,7 +648,13 @@ public SafeFuture<SendSignedBlockResult> sendSignedBlock(
  maybeBlindedBlockContainer.getSlot());
  return blockPublisher
  .sendSignedBlock(
- maybeBlindedBlockContainer, broadcastValidationLevel, blockPublishingPerformance)
+ maybeBlindedBlockContainer,
+ // do only EQUIVOCATION validation when GOSSIP validation has been requested and the
+ // block has been locally created
+ broadcastValidationLevel == GOSSIP && isLocallyCreatedBlock(maybeBlindedBlockContainer)
+ ? EQUIVOCATION
+ : broadcastValidationLevel,
+ blockPublishingPerformance)
  .exceptionally(
  ex -> {
  final String reason = getRootCauseMessage(ex);
@@ -842,6 +860,13 @@ private List<ProposerDuty> getProposalSlotsForEpoch(final BeaconState state, fin
  return proposerSlots;
  }
 
+ private boolean isLocallyCreatedBlock(final SignedBlockContainer blockContainer) {
+ final Bytes32 blockRoot = blockContainer.getSignedBlock().getMessage().getRoot();
+ final Bytes32 locallyCreatedBlockRoot =
+ createdBlockRootsBySlotCache.get(blockContainer.getSlot());
+ return Objects.equals(blockRoot, locallyCreatedBlockRoot);
+ }
+
  @Override
  public SafeFuture<Optional<List<BeaconCommitteeSelectionProof>>> getBeaconCommitteeSelectionProof(
  final List<BeaconCommitteeSelectionProof> requests) {

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandlerTest.java

@@ -33,6 +33,8 @@
 import static tech.pegasys.teku.infrastructure.async.SafeFutureAssert.safeJoin;
 import static tech.pegasys.teku.infrastructure.unsigned.UInt64.ONE;
 import static tech.pegasys.teku.infrastructure.unsigned.UInt64.ZERO;
+import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.EQUIVOCATION;
+import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.GOSSIP;
 import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.NOT_REQUIRED;
 import static tech.pegasys.teku.spec.logic.common.statetransition.results.BlockImportResult.FailureReason.DOES_NOT_DESCEND_FROM_LATEST_FINALIZED;
 
@@ -892,6 +894,56 @@ public void sendSignedBlock_shouldConvertKnownBlockResult() {
  assertThat(result).isCompletedWithValue(SendSignedBlockResult.success(block.getRoot()));
  }
 
+ @Test
+ public void
+ sendSignedBlock_shouldOnlyDoEquivocationValidationIfBlockIsLocallyCreatedAngGossipValidationRequested() {
+ // creating a block first in order to cache the block root
+ final UInt64 newSlot = UInt64.valueOf(25);
+ final BeaconState blockSlotState = dataStructureUtil.randomBeaconState(newSlot);
+ final BLSSignature randaoReveal = dataStructureUtil.randomSignature();
+ final BlockContainerAndMetaData blockContainerAndMetaData =
+ dataStructureUtil.randomBlockContainerAndMetaData(newSlot);
+
+ when(chainDataClient.getStateForBlockProduction(newSlot, false))
+ .thenReturn(SafeFuture.completedFuture(Optional.of(blockSlotState)));
+ when(blockFactory.createUnsignedBlock(
+ blockSlotState,
+ newSlot,
+ randaoReveal,
+ Optional.empty(),
+ Optional.of(false),
+ Optional.of(ONE),
+ BlockProductionPerformance.NOOP))
+ .thenReturn(SafeFuture.completedFuture(blockContainerAndMetaData));
+
+ assertThat(
+ validatorApiHandler.createUnsignedBlock(
+ newSlot, randaoReveal, Optional.empty(), Optional.of(false), Optional.of(ONE)))
+ .isCompleted();
+
+ final SignedBeaconBlock block =
+ dataStructureUtil
+ .getSpec()
+ .atSlot(newSlot)
+ .getSchemaDefinitions()
+ .getSignedBeaconBlockSchema()
+ .create(
+ blockContainerAndMetaData.blockContainer().getBlock(),
+ dataStructureUtil.randomSignature());
+
+ when(blockImportChannel.importBlock(eq(block), any()))
+ .thenReturn(prepareBlockImportResult(BlockImportResult.successful(block)));
+
+ // require GOSSIP validation
+ final SafeFuture<SendSignedBlockResult> result =
+ validatorApiHandler.sendSignedBlock(block, GOSSIP);
+
+ assertThat(result).isCompletedWithValue(SendSignedBlockResult.success(block.getRoot()));
+
+ // for locally created blocks, the validation level should have been changed to EQUIVOCATION
+ verify(blockImportChannel).importBlock(block, EQUIVOCATION);
+ }
+
  @Test
  public void sendSignedBlock_shouldConvertBlockContentsSuccessfulResult() {
  setupDeneb();

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/validator/BroadcastValidationLevel.java

@@ -15,6 +15,7 @@
 
 public enum BroadcastValidationLevel {
  NOT_REQUIRED,
+ EQUIVOCATION,
  GOSSIP,
  CONSENSUS,
  CONSENSUS_AND_EQUIVOCATION

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlockBroadcastValidator.java

@@ -58,7 +58,7 @@ enum BroadcastValidationResult {
  SUCCESS,
  GOSSIP_FAILURE,
  CONSENSUS_FAILURE,
- FINAL_EQUIVOCATION_FAILURE;
+ EQUIVOCATION_FAILURE;
 
  public boolean isFailure() {
  return this != SUCCESS;

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlockBroadcastValidatorImpl.java

@@ -13,10 +13,11 @@
 
 package tech.pegasys.teku.statetransition.validation;
 
+import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.EQUIVOCATION;
 import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.GOSSIP;
 import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.NOT_REQUIRED;
 import static tech.pegasys.teku.statetransition.validation.BlockBroadcastValidator.BroadcastValidationResult.CONSENSUS_FAILURE;
-import static tech.pegasys.teku.statetransition.validation.BlockBroadcastValidator.BroadcastValidationResult.FINAL_EQUIVOCATION_FAILURE;
+import static tech.pegasys.teku.statetransition.validation.BlockBroadcastValidator.BroadcastValidationResult.EQUIVOCATION_FAILURE;
 import static tech.pegasys.teku.statetransition.validation.BlockBroadcastValidator.BroadcastValidationResult.GOSSIP_FAILURE;
 import static tech.pegasys.teku.statetransition.validation.BlockBroadcastValidator.BroadcastValidationResult.SUCCESS;
 
@@ -59,10 +60,10 @@ public void onConsensusValidationSucceeded() {
  @Override
  public void attachToBlockImport(final SafeFuture<BlockImportResult> blockImportResult) {
  switch (broadcastValidationLevel) {
- case NOT_REQUIRED, GOSSIP:
- // GOSSIP validation isn't dependent on block import result,
+ case NOT_REQUIRED, EQUIVOCATION, GOSSIP:
+ // EQUIVOCATION/GOSSIP validation isn't dependent on block import result,
  // so not propagating exceptions to consensusValidationSuccessResult allow blocks\blobs
- // to be published even in case block import fails before gossip validation completes
+ // to be published even in case block import fails before the validation completes
  return;
  case CONSENSUS, CONSENSUS_AND_EQUIVOCATION:
  // Any successful block import will be considered as a consensus validation success, but
@@ -87,7 +88,20 @@ private void buildValidationPipeline(final SignedBeaconBlock block) {
  return;
  }
 
- // GOSSIP only validation
+ // EQUIVOCATION only validation
+ if (broadcastValidationLevel == EQUIVOCATION) {
+ final BroadcastValidationResult validationResult;
+ if (isEquivocatingBlock(block)) {
+ validationResult = EQUIVOCATION_FAILURE;
+ } else {
+ validationResult = SUCCESS;
+ }
+ broadcastValidationResult.complete(validationResult);
+ consensusValidationSuccessResult.cancel(true);
+ return;
+ }
+
+ // GOSSIP only validation (includes EQUIVOCATION validation)
  SafeFuture<BroadcastValidationResult> validationPipeline =
  blockGossipValidator
  .validate(block, true)
@@ -128,7 +142,7 @@ private void buildValidationPipeline(final SignedBeaconBlock block) {
  return;
  }
 
- // GOSSIP, CONSENSUS and additional EQUIVOCATION validation
+ // GOSSIP, CONSENSUS and final EQUIVOCATION validation
  validationPipeline
  .thenApply(
  broadcastValidationResult -> {
@@ -138,14 +152,18 @@ private void buildValidationPipeline(final SignedBeaconBlock block) {
  }
 
  // perform final equivocation validation
- if (blockGossipValidator
- .performBlockEquivocationCheck(block)
- .equals(EquivocationCheckResult.EQUIVOCATING_BLOCK_FOR_SLOT_PROPOSER)) {
- return FINAL_EQUIVOCATION_FAILURE;
+ if (isEquivocatingBlock(block)) {
+ return EQUIVOCATION_FAILURE;
  }
 
  return SUCCESS;
  })
  .propagateTo(broadcastValidationResult);
  }
+
+ private boolean isEquivocatingBlock(final SignedBeaconBlock block) {
+ return blockGossipValidator
+ .performBlockEquivocationCheck(block)
+ .equals(EquivocationCheckResult.EQUIVOCATING_BLOCK_FOR_SLOT_PROPOSER);
+ }
 }

ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/validation/BlockBroadcastValidatorTest.java

@@ -17,13 +17,15 @@
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.CONSENSUS_AND_EQUIVOCATION;
+import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.EQUIVOCATION;
 import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.GOSSIP;
 import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.NOT_REQUIRED;
 import static tech.pegasys.teku.statetransition.validation.BlockBroadcastValidator.BroadcastValidationResult.CONSENSUS_FAILURE;
-import static tech.pegasys.teku.statetransition.validation.BlockBroadcastValidator.BroadcastValidationResult.FINAL_EQUIVOCATION_FAILURE;
+import static tech.pegasys.teku.statetransition.validation.BlockBroadcastValidator.BroadcastValidationResult.EQUIVOCATION_FAILURE;
 import static tech.pegasys.teku.statetransition.validation.BlockBroadcastValidator.BroadcastValidationResult.GOSSIP_FAILURE;
 import static tech.pegasys.teku.statetransition.validation.BlockBroadcastValidator.BroadcastValidationResult.SUCCESS;
 import static tech.pegasys.teku.statetransition.validation.ValidationResultCode.ValidationResultSubCode.IGNORE_ALREADY_SEEN;
@@ -56,6 +58,32 @@ public class BlockBroadcastValidatorTest {
 
  final SafeFuture<BlockImportResult> blockImportResult = new SafeFuture<>();
 
+ @Test
+ public void shouldReturnSuccessWhenValidationIsEquivocationAndBlockIsNotEquivocating() {
+ when(blockGossipValidator.performBlockEquivocationCheck(block))
+ .thenReturn(EquivocationCheckResult.FIRST_BLOCK_FOR_SLOT_PROPOSER);
+
+ prepareBlockBroadcastValidator(EQUIVOCATION);
+
+ assertThat(blockBroadcastValidator.getResult())
+ .isCompletedWithValueMatching(result -> result.equals(SUCCESS));
+ verify(blockGossipValidator).performBlockEquivocationCheck(block);
+ verifyNoMoreInteractions(blockGossipValidator);
+ }
+
+ @Test
+ public void shouldReturnEquivocationFailureWhenValidationIsEquivocationAndBlockIsEquivocating() {
+ when(blockGossipValidator.performBlockEquivocationCheck(block))
+ .thenReturn(EquivocationCheckResult.EQUIVOCATING_BLOCK_FOR_SLOT_PROPOSER);
+
+ prepareBlockBroadcastValidator(EQUIVOCATION);
+
+ assertThat(blockBroadcastValidator.getResult())
+ .isCompletedWithValueMatching(result -> result.equals(EQUIVOCATION_FAILURE));
+ verify(blockGossipValidator).performBlockEquivocationCheck(block);
+ verifyNoMoreInteractions(blockGossipValidator);
+ }
+
  @Test
  public void shouldReturnSuccessWhenValidationIsGossipAndGossipValidationReturnsAccept() {
  when(blockGossipValidator.validate(eq(block), eq(true)))
@@ -109,10 +137,19 @@ public void shouldReturnGossipFailureImmediatelyWhenGossipValidationIsNotAccept(
 
  if (broadcastValidation == NOT_REQUIRED) {
  prepareBlockBroadcastValidator(broadcastValidation);
+ assertThat(blockBroadcastValidator.getResult())
+ .isCompletedWithValueMatching(result -> result.equals(SUCCESS));
+ verifyNoInteractions(blockGossipValidator);
+ return;
+ }
 
+ if (broadcastValidation == EQUIVOCATION) {
+ when(blockGossipValidator.performBlockEquivocationCheck(block))
+ .thenReturn(EquivocationCheckResult.FIRST_BLOCK_FOR_SLOT_PROPOSER);
+ prepareBlockBroadcastValidator(broadcastValidation);
  assertThat(blockBroadcastValidator.getResult())
  .isCompletedWithValueMatching(result -> result.equals(SUCCESS));
- verifyNoMoreInteractions(blockGossipValidator);
+ verify(blockGossipValidator).performBlockEquivocationCheck(block);
  return;
  }
 
@@ -193,7 +230,7 @@ public void shouldReturnFinalEquivocationFailureOnlyForEquivocatingBlocks(
  result -> {
  if (equivocationCheckResult.equals(
  EquivocationCheckResult.EQUIVOCATING_BLOCK_FOR_SLOT_PROPOSER)) {
- return result.equals(FINAL_EQUIVOCATION_FAILURE);
+ return result.equals(EQUIVOCATION_FAILURE);
  }
  return result.equals(SUCCESS);
  });

validator/client/src/main/java/tech/pegasys/teku/validator/client/duties/BlockProductionDuty.java

@@ -155,7 +155,7 @@ private SafeFuture<SignedBlockContainer> signBlockContainer(
 
  private SafeFuture<DutyResult> sendBlock(final SignedBlockContainer signedBlockContainer) {
  return validatorApiChannel
- .sendSignedBlock(signedBlockContainer, BroadcastValidationLevel.NOT_REQUIRED)
+ .sendSignedBlock(signedBlockContainer, BroadcastValidationLevel.GOSSIP)
  .thenApply(
  result -> {
  if (result.isPublished()) {