Initial cut of late block reorg

ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java

@@ -49,12 +49,16 @@ public class Eth2NetworkConfiguration {
   private static final int DEFAULT_STARTUP_TARGET_PEER_COUNT = 5;
   private static final int DEFAULT_STARTUP_TIMEOUT_SECONDS = 30;
 
+  public static final boolean DEFAULT_FORK_CHOICE_UPDATE_HEAD_ON_BLOCK_IMPORT_ENABLED = false;
+
   public static final boolean DEFAULT_FORK_CHOICE_LATE_BLOCK_REORG_ENABLED = false;
 
   public static final boolean DEFAULT_FORK_CHOICE_UPDATED_ALWAYS_SEND_PAYLOAD_ATTRIBUTES = false;
 
   public static final boolean DEFAULT_ALLOW_SYNC_OUTSIDE_WEAK_SUBJECTIVITY_PERIOD = false;
 
+  public static final boolean DEFAULT_FORK_CHOICE_VALIDATOR_IS_PROPOSER_ALWAYS_ENABLED = false;
+
   public static final int DEFAULT_ASYNC_P2P_MAX_THREADS = 10;
 
   public static final int DEFAULT_ASYNC_P2P_MAX_QUEUE = DEFAULT_MAX_QUEUE_SIZE;
@@ -99,6 +103,8 @@ public class Eth2NetworkConfiguration {
   private final boolean forkChoiceLateBlockReorgEnabled;
   private final boolean forkChoiceUpdatedAlwaysSendPayloadAttributes;
 
+  private final boolean forkChoiceValidatorIsProposerAlwaysEnabled;
+
   private Eth2NetworkConfiguration(
       final Spec spec,
       final String constants,
@@ -123,6 +129,7 @@ private Eth2NetworkConfiguration(
       final int asyncBeaconChainMaxThreads,
       final int asyncBeaconChainMaxQueue,
       final boolean forkChoiceLateBlockReorgEnabled,
+      final boolean forkChoiceValidatorIsProposerAlwaysEnabled,
       final boolean forkChoiceUpdatedAlwaysSendPayloadAttributes) {
     this.spec = spec;
     this.constants = constants;
@@ -150,6 +157,7 @@ private Eth2NetworkConfiguration(
     this.asyncBeaconChainMaxThreads = asyncBeaconChainMaxThreads;
     this.asyncBeaconChainMaxQueue = asyncBeaconChainMaxQueue;
     this.forkChoiceLateBlockReorgEnabled = forkChoiceLateBlockReorgEnabled;
+    this.forkChoiceValidatorIsProposerAlwaysEnabled = forkChoiceValidatorIsProposerAlwaysEnabled;
     this.forkChoiceUpdatedAlwaysSendPayloadAttributes =
         forkChoiceUpdatedAlwaysSendPayloadAttributes;
   }
@@ -261,6 +269,10 @@ public boolean isForkChoiceUpdatedAlwaysSendPayloadAttributes() {
     return forkChoiceUpdatedAlwaysSendPayloadAttributes;
   }
 
+  public boolean isForkChoiceValidatorIsProposerAlwaysEnabled() {
+    return forkChoiceValidatorIsProposerAlwaysEnabled;
+  }
+
   @Override
   public String toString() {
     return constants;
@@ -298,6 +310,8 @@ public static class Builder {
     private boolean forkChoiceLateBlockReorgEnabled = DEFAULT_FORK_CHOICE_LATE_BLOCK_REORG_ENABLED;
     private boolean forkChoiceUpdatedAlwaysSendPayloadAttributes =
         DEFAULT_FORK_CHOICE_UPDATED_ALWAYS_SEND_PAYLOAD_ATTRIBUTES;
+    private boolean forkChoiceValidatorIsProposerAlwaysEnabled =
+        DEFAULT_FORK_CHOICE_VALIDATOR_IS_PROPOSER_ALWAYS_ENABLED;
 
     public void spec(Spec spec) {
       this.spec = spec;
@@ -380,6 +394,7 @@ public Eth2NetworkConfiguration build() {
           asyncBeaconChainMaxThreads,
           asyncBeaconChainMaxQueue,
           forkChoiceLateBlockReorgEnabled,
+          forkChoiceValidatorIsProposerAlwaysEnabled,
           forkChoiceUpdatedAlwaysSendPayloadAttributes);
     }
 
@@ -855,5 +870,11 @@ public Builder forkChoiceUpdatedAlwaysSendPayloadAttributes(
           forkChoiceUpdatedAlwaysSendPayloadAttributes;
       return this;
     }
+
+    public Builder forkChoiceValidatorIsProposerAlwaysEnabled(
+        boolean forkChoiceValidatorIsProposerAlwaysEnabled) {
+      this.forkChoiceValidatorIsProposerAlwaysEnabled = forkChoiceValidatorIsProposerAlwaysEnabled;
+      return this;
+    }
   }
 }

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java

@@ -758,10 +758,9 @@ private void reportInvalidBlock(final SignedBeaconBlock block, final BlockImport
 
   private void notifyForkChoiceUpdatedAndOptimisticSyncingChanged(
       final Optional<UInt64> proposingSlot) {
-    final ForkChoiceState forkChoiceState = forkChoiceStateProvider.getForkChoiceStateSync();
+    final ForkChoiceState forkChoiceState = forkChoiceStateProvider.getForkChoiceStateSync(false);
 
-    forkChoiceNotifier.onForkChoiceUpdated(forkChoiceState, proposingSlot);
-    getProposerHeadSelectedCounter.labels("fork_choice").inc();
+    notifyForkChoiceUpdated(forkChoiceState, proposingSlot);
 
     if (optimisticSyncing
         .map(oldValue -> !oldValue.equals(forkChoiceState.isHeadOptimistic()))
@@ -772,6 +771,58 @@ private void notifyForkChoiceUpdatedAndOptimisticSyncingChanged(
     }
   }
 
+  @SuppressWarnings("AlreadyChecked")
+  private void notifyForkChoiceUpdated(
+      final ForkChoiceState forkChoiceState, final Optional<UInt64> proposingSlot) {
+    if (!forkChoiceLateBlockReorgEnabled) {
+      forkChoiceNotifier.onForkChoiceUpdated(forkChoiceState, proposingSlot);
+      return;
+    }
+    // late block reorg logic
+    final boolean shouldOverrideForkChoice =
+        recentChainData.shouldOverrideForkChoiceUpdate(forkChoiceState.getHeadBlockRoot());
+    final Optional<Bytes32> maybeProposerHead =
+        proposingSlot.map(
+            slot ->
+                recentChainData.getProposerHead(
+                    forkChoiceState.getHeadBlockRoot(), proposingSlot.get()));
+
+    if (!shouldOverrideForkChoice) {
+      LOG.debug(
+          "Fork Choice Notify shouldOverrideForkChoice {}, getProposerHead {}, chose fork choice head {}:({})",
+          () -> shouldOverrideForkChoice,
+          () -> maybeProposerHead,
+          forkChoiceState::getHeadBlockRoot,
+          forkChoiceState::getHeadBlockSlot);
+      forkChoiceNotifier.onForkChoiceUpdated(forkChoiceState, proposingSlot);
+      maybeProposerHead.ifPresent(
+          proposerHead -> getProposerHeadSelectedCounter.labels("fork_choice").inc());
+      return;
+    } else if (maybeProposerHead.isPresent()
+        && maybeProposerHead.get().equals(forkChoiceState.getHeadBlockRoot())) {
+      LOG.debug(
+          "Fork Choice Notify shouldOverrideForkChoice {}, getProposerHead {}, chose head {}:({})",
+          () -> shouldOverrideForkChoice,
+          () -> maybeProposerHead,
+          forkChoiceState::getHeadBlockRoot,
+          forkChoiceState::getHeadBlockSlot);
+      getProposerHeadSelectedCounter.labels("head").inc();
+      forkChoiceNotifier.onForkChoiceUpdated(forkChoiceState, proposingSlot);
+      return;
+    }
+    final ForkChoiceState parentForkChoiceState =
+        forkChoiceStateProvider.getForkChoiceStateSync(true);
+    LOG.debug(
+        "Fork Choice Notify shouldOverrideForkChoice {}, fork choice head {}:({}), chose parent {}:({})",
+        () -> shouldOverrideForkChoice,
+        forkChoiceState::getHeadBlockRoot,
+        forkChoiceState::getHeadBlockSlot,
+        parentForkChoiceState::getHeadBlockRoot,
+        parentForkChoiceState::getHeadBlockSlot);
+    getProposerHeadSelectedCounter.labels("parent").inc();
+    forkChoiceNotifier.onForkChoiceUpdated(parentForkChoiceState, proposingSlot);
+  }
+
   private void applyVotesFromBlock(
       final ForkChoiceStrategy forkChoiceStrategy,
       final UInt64 currentEpoch,

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceNotifierImpl.java

@@ -25,6 +25,7 @@
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.config.SpecConfig;
 import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadContext;
+import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 import tech.pegasys.teku.spec.executionlayer.ExecutionLayerChannel;
 import tech.pegasys.teku.spec.executionlayer.ForkChoiceState;
 import tech.pegasys.teku.statetransition.forkchoice.ForkChoiceUpdatedResultSubscriber.ForkChoiceUpdatedResultNotification;
@@ -171,7 +172,7 @@ private SafeFuture<Optional<ExecutionPayloadContext>> internalGetPayloadId(
           .getForkChoiceStateAsync()
           .thenCombine(
               proposersDataManager.calculatePayloadBuildingAttributes(
-                  blockSlot, inSync, localForkChoiceUpdateData, true),
+                  blockSlot, inSync, localForkChoiceUpdateData, true, Optional.empty()),
               (forkChoiceState, payloadBuildingAttributes) -> {
                 forkChoiceUpdateData =
                     localForkChoiceUpdateData
@@ -207,7 +208,7 @@ private void internalUpdatePreparableProposers() {
     final UInt64 currentSlot = recentChainData.getCurrentSlot().orElse(SpecConfig.GENESIS_SLOT);
 
     // Update payload attributes in case we now need to propose the next block
-    updatePayloadAttributes(currentSlot.plus(1));
+    updatePayloadAttributes(currentSlot.plus(1), Optional.empty());
   }
 
   private void internalForkChoiceUpdated(
@@ -223,7 +224,8 @@ private void internalForkChoiceUpdated(
     final Optional<UInt64> attributesSlot =
         proposingSlot.or(() -> recentChainData.getCurrentSlot().map(UInt64::increment));
 
-    attributesSlot.ifPresent(this::updatePayloadAttributes);
+    attributesSlot.ifPresent(
+        slot -> updatePayloadAttributesWithForkChoiceState(slot, Optional.of(forkChoiceState)));
 
     sendForkChoiceUpdated();
   }
@@ -234,7 +236,7 @@ private void internalAttestationsDue(final UInt64 slot) {
     LOG.debug("internalAttestationsDue slot {}", slot);
 
     // Assume `slot` is empty and check if we need to prepare to propose in the next slot
-    updatePayloadAttributes(slot.plus(1));
+    updatePayloadAttributes(slot.plus(1), Optional.empty());
   }
 
   private void sendForkChoiceUpdated() {
@@ -251,14 +253,38 @@ private void sendForkChoiceUpdated() {
                         forkChoiceUpdatedResultFuture)));
   }
 
-  private void updatePayloadAttributes(final UInt64 blockSlot) {
+  private void updatePayloadAttributesWithForkChoiceState(
+      final UInt64 blockSlot, final Optional<ForkChoiceState> maybeForkChoiceState) {
+    final SafeFuture<Optional<BeaconState>> maybeFutureState;
+    final Optional<Bytes32> maybeRoot = recentChainData.getBestBlockRoot();
+    if (maybeForkChoiceState.isPresent()
+        && maybeRoot.isPresent()
+        && !maybeRoot.get().equals(maybeForkChoiceState.get().getHeadBlockRoot())) {
+      // if we know the root we're building on isn't the head root, supply the state
+      maybeFutureState =
+          recentChainData.retrieveBlockState(maybeForkChoiceState.get().getHeadBlockRoot());
+    } else {
+      maybeFutureState = SafeFuture.completedFuture(Optional.empty());
+    }
+    maybeFutureState
+        .thenAccept(maybeState -> updatePayloadAttributes(blockSlot, maybeState))
+        .finish(
+            error ->
+                LOG.debug(
+                    "Failed to retrieve state to calculate payload attributes for slot {}",
+                    blockSlot,
+                    error));
+  }
+
+  private void updatePayloadAttributes(
+      final UInt64 blockSlot, final Optional<BeaconState> maybeState) {
     LOG.debug("updatePayloadAttributes blockSlot {}", blockSlot);
 
     forkChoiceUpdateData
         .withPayloadBuildingAttributesAsync(
             () ->
                 proposersDataManager.calculatePayloadBuildingAttributes(
-                    blockSlot, inSync, forkChoiceUpdateData, false),
+                    blockSlot, inSync, forkChoiceUpdateData, false, maybeState),
             eventThread)
         .thenAccept(
             newForkChoiceUpdateData -> {

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceStateProvider.java

@@ -28,21 +28,22 @@ public ForkChoiceStateProvider(EventThread forkChoiceExecutor, RecentChainData r
   }
 
   public SafeFuture<ForkChoiceState> getForkChoiceStateAsync() {
-    return forkChoiceExecutor.execute(this::internalGetForkChoiceState);
+    return forkChoiceExecutor.execute(() -> internalGetForkChoiceState(false));
   }
 
-  public ForkChoiceState getForkChoiceStateSync() {
+  public ForkChoiceState getForkChoiceStateSync(final boolean isForkChoiceOverrideHead) {
     forkChoiceExecutor.checkOnEventThread();
-    return internalGetForkChoiceState();
+    return internalGetForkChoiceState(isForkChoiceOverrideHead);
   }
 
-  private ForkChoiceState internalGetForkChoiceState() {
+  private ForkChoiceState internalGetForkChoiceState(final boolean isForkChoiceOverrideHead) {
     return recentChainData
         .getUpdatableForkChoiceStrategy()
         .orElseThrow()
         .getForkChoiceState(
             recentChainData.getCurrentEpoch().orElseThrow(),
             recentChainData.getJustifiedCheckpoint().orElseThrow(),
-            recentChainData.getFinalizedCheckpoint().orElseThrow());
+            recentChainData.getFinalizedCheckpoint().orElseThrow(),
+            isForkChoiceOverrideHead);
   }
 }

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ProposersDataManager.java

@@ -40,6 +40,8 @@
 import tech.pegasys.teku.spec.executionlayer.ExecutionLayerChannel;
 import tech.pegasys.teku.spec.executionlayer.ForkChoiceState;
 import tech.pegasys.teku.spec.executionlayer.PayloadBuildingAttributes;
+import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.EpochProcessingException;
+import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.SlotProcessingException;
 import tech.pegasys.teku.storage.client.ChainHead;
 import tech.pegasys.teku.storage.client.RecentChainData;
 
@@ -189,7 +191,8 @@ public SafeFuture<Optional<PayloadBuildingAttributes>> calculatePayloadBuildingA
       final UInt64 blockSlot,
       final boolean inSync,
       final ForkChoiceUpdateData forkChoiceUpdateData,
-      final boolean mandatory) {
+      final boolean mandatory,
+      final Optional<BeaconState> maybeState) {
     eventThread.checkOnEventThread();
     if (!inSync) {
       // We don't produce blocks while syncing so don't bother preparing the payload
@@ -207,11 +210,21 @@ public SafeFuture<Optional<PayloadBuildingAttributes>> calculatePayloadBuildingA
     final UInt64 epoch = spec.computeEpochAtSlot(blockSlot);
     final ForkChoiceState forkChoiceState = forkChoiceUpdateData.getForkChoiceState();
     final Bytes32 currentHeadBlockRoot = forkChoiceState.getHeadBlockRoot();
+    if (maybeState.isPresent()) {
+      try {
+        final BeaconState preState = spec.processSlots(maybeState.get(), blockSlot);
+        return SafeFuture.completedFuture(
+            calculatePayloadBuildingAttributes(
+                currentHeadBlockRoot, blockSlot, epoch, Optional.of(preState), mandatory));
+      } catch (SlotProcessingException | EpochProcessingException e) {
+        LOG.error("Failed to process slots to get state current at blockSlot {}", blockSlot, e);
+      }
+    }
     return getStateInEpoch(epoch)
         .thenApplyAsync(
-            maybeState ->
+            maybeStateInEpoch ->
                 calculatePayloadBuildingAttributes(
-                    currentHeadBlockRoot, blockSlot, epoch, maybeState, mandatory),
+                    currentHeadBlockRoot, blockSlot, epoch, maybeStateInEpoch, mandatory),
             eventThread);
   }
 

ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceNotifierTest.java

@@ -331,7 +331,7 @@ void onForkChoiceUpdated_shouldNotSendNotificationOfOutOfOrderPayloadBuildingAtt
               return deferredResponseA;
             })
         .when(proposersDataManager)
-        .calculatePayloadBuildingAttributes(any(), anyBoolean(), any(), anyBoolean());
+        .calculatePayloadBuildingAttributes(any(), anyBoolean(), any(), anyBoolean(), any());
 
     notifyForkChoiceUpdated(forkChoiceState); // calculate attributes for slot 2
 
@@ -341,7 +341,7 @@ void onForkChoiceUpdated_shouldNotSendNotificationOfOutOfOrderPayloadBuildingAtt
     // forward to real method call
     doAnswer(InvocationOnMock::callRealMethod)
         .when(proposersDataManager)
-        .calculatePayloadBuildingAttributes(any(), anyBoolean(), any(), anyBoolean());
+        .calculatePayloadBuildingAttributes(any(), anyBoolean(), any(), anyBoolean(), any());
 
     storageSystem
         .chainUpdater()
@@ -441,7 +441,7 @@ void onForkChoiceUpdated_shouldNotSendNotificationWithOldPayloadBuildingAttribut
               return deferredResponseA;
             })
         .when(proposersDataManager)
-        .calculatePayloadBuildingAttributes(any(), anyBoolean(), any(), anyBoolean());
+        .calculatePayloadBuildingAttributes(any(), anyBoolean(), any(), anyBoolean(), any());
 
     notifyForkChoiceUpdated(forkChoiceStateSlot3); // calculate attributes for slot 2
 

storage/src/main/java/tech/pegasys/teku/storage/protoarray/ForkChoiceStrategy.java

@@ -184,11 +184,13 @@ public List<ProtoNodeData> getChainHeads(final boolean includeNonViableHeads) {
   public ForkChoiceState getForkChoiceState(
       final UInt64 currentEpoch,
       final Checkpoint justifiedCheckpoint,
-      final Checkpoint finalizedCheckpoint) {
+      final Checkpoint finalizedCheckpoint,
+      final boolean isForkChoiceOverrideHead) {
     protoArrayLock.readLock().lock();
     try {
       final ProtoNode headNode =
           protoArray.findOptimisticHead(currentEpoch, justifiedCheckpoint, finalizedCheckpoint);
+      final Optional<ProtoNode> maybeParentNode = protoArray.getProtoNode(headNode.getParentRoot());
       final UInt64 headExecutionBlockNumber = headNode.getExecutionBlockNumber();
       final Bytes32 headExecutionBlockHash = headNode.getExecutionBlockHash();
       final Bytes32 justifiedExecutionHash =
@@ -201,6 +203,17 @@ public ForkChoiceState getForkChoiceState(
               .getProtoNode(finalizedCheckpoint.getRoot())
               .map(ProtoNode::getExecutionBlockHash)
               .orElse(Bytes32.ZERO);
+      if (isForkChoiceOverrideHead && maybeParentNode.isPresent()) {
+        final ProtoNode parentNode = maybeParentNode.get();
+        return new ForkChoiceState(
+            parentNode.getBlockRoot(),
+            parentNode.getBlockSlot(),
+            parentNode.getExecutionBlockNumber(),
+            parentNode.getExecutionBlockHash(),
+            justifiedExecutionHash,
+            finalizedExecutionHash,
+            headNode.isOptimistic() || !protoArray.nodeIsViableForHead(headNode));
+      }
       return new ForkChoiceState(
           headNode.getBlockRoot(),
           headNode.getBlockSlot(),
@@ -289,10 +302,10 @@ public Optional<UInt64> executionBlockNumber(final Bytes32 blockRoot) {
   }
 
   @Override
-  public Optional<Bytes32> executionBlockHash(final Bytes32 blockRoot) {
+  public Optional<Bytes32> executionBlockHash(final Bytes32 beaconBlockRoot) {
     protoArrayLock.readLock().lock();
     try {
-      return getProtoNode(blockRoot).map(ProtoNode::getExecutionBlockHash);
+      return getProtoNode(beaconBlockRoot).map(ProtoNode::getExecutionBlockHash);
     } finally {
       protoArrayLock.readLock().unlock();
     }