Merge branch 'hyperledger:main' into issue-7047-optional-jemalloc

.github/workflows/acceptance-tests.yml

@@ -11,7 +11,7 @@ concurrency:
  cancel-in-progress: true
 
 env:
- GRADLE_OPTS: "-Xmx6g"
+ GRADLE_OPTS: "-Xmx7g"
  total-runners: 12
 
 jobs:

CHANGELOG.md

@@ -9,12 +9,13 @@
 
 ### Additions and Improvements
 - Remove privacy test classes support [#7569](https://github.com/hyperledger/besu/pull/7569)
+- Add Blob Transaction Metrics [#7622](https://github.com/hyperledger/besu/pull/7622)
 
 ### Bug fixes
 - Fix mounted data path directory permissions for besu user [#7575](https://github.com/hyperledger/besu/pull/7575)
 - Fix for `debug_traceCall` to handle transactions without specified gas price. [#7510](https://github.com/hyperledger/besu/pull/7510)
 - Corrects a regression where custom plugin services are not initialized correctly. [#7625](https://github.com/hyperledger/besu/pull/7625)
-
+- Fix for IBFT2 chains using the BONSAI DB format [#7631](https://github.com/hyperledger/besu/pull/7631)
 
 ## 24.9.1
 

MAINTAINERS.md

@@ -28,7 +28,6 @@
 | Matthew Whitehead| matthew1001 | matthew.whitehead |
 | Meredith Baxter | mbaxter | mbaxter |
 | Stefan Pingel | pinges | pinges |
-| Danno Ferrin | shemnon | shemnon |
 | Simon Dudley | siladu | siladu |
 | Usman Saleem | usmansaleem | usmansaleem |
 
@@ -52,6 +51,7 @@
 | Rai Sur | RatanRSur | ratanraisur |
 | Rob Dawson | rojotek | RobDawson |
 | Sajida Zouarhi | sajz | SajidaZ |
+| Danno Ferrin | shemnon | shemnon |
 | Taccat Isid | taccatisid | taccatisid |
 | Tim Beiko | timbeiko | timbeiko |
 | Vijay Michalik | vmichalik | VijayMichalik |

besu/src/main/java/org/hyperledger/besu/controller/IbftBesuControllerBuilder.java

@@ -74,6 +74,7 @@
 import org.hyperledger.besu.plugin.services.BesuEvents;
 import org.hyperledger.besu.util.Subscribers;
 
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -182,7 +183,10 @@ protected MiningCoordinator createMiningCoordinator(
  Util.publicKeyToAddress(nodeKey.getPublicKey()),
  proposerSelector,
  uniqueMessageMulticaster,
- new RoundTimer(bftEventQueue, bftConfig.getRequestTimeoutSeconds(), bftExecutors),
+ new RoundTimer(
+ bftEventQueue,
+ Duration.ofSeconds(bftConfig.getRequestTimeoutSeconds()),
+ bftExecutors),
  new BlockTimer(bftEventQueue, forksSchedule, bftExecutors, clock),
  blockCreatorFactory,
  clock);

besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java

@@ -84,6 +84,7 @@
 import org.hyperledger.besu.plugin.services.BesuEvents;
 import org.hyperledger.besu.util.Subscribers;
 
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -222,7 +223,10 @@ protected MiningCoordinator createMiningCoordinator(
  Util.publicKeyToAddress(nodeKey.getPublicKey()),
  proposerSelector,
  uniqueMessageMulticaster,
- new RoundTimer(bftEventQueue, qbftConfig.getRequestTimeoutSeconds(), bftExecutors),
+ new RoundTimer(
+ bftEventQueue,
+ Duration.ofSeconds(qbftConfig.getRequestTimeoutSeconds()),
+ bftExecutors),
  new BlockTimer(bftEventQueue, qbftForksSchedule, bftExecutors, clock),
  blockCreatorFactory,
  clock);

config/src/main/java/org/hyperledger/besu/config/BftConfigOptions.java

@@ -37,6 +37,13 @@ public interface BftConfigOptions {
  */
  int getBlockPeriodSeconds();
 
+ /**
+ * Gets block period milliseconds. For TESTING only. If set then blockperiodseconds is ignored.
+ *
+ * @return the block period milliseconds
+ */
+ long getBlockPeriodMilliseconds();
+
  /**
  * Gets request timeout seconds.
  *

config/src/main/java/org/hyperledger/besu/config/BftFork.java

@@ -21,6 +21,7 @@
 import java.util.Locale;
 import java.util.Optional;
 import java.util.OptionalInt;
+import java.util.OptionalLong;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -40,6 +41,9 @@ public class BftFork implements Fork {
  /** The constant BLOCK_PERIOD_SECONDS_KEY. */
  public static final String BLOCK_PERIOD_SECONDS_KEY = "blockperiodseconds";
 
+ /** The constant BLOCK_PERIOD_MILLISECONDS_KEY. */
+ public static final String BLOCK_PERIOD_MILLISECONDS_KEY = "xblockperiodmilliseconds";
+
  /** The constant BLOCK_REWARD_KEY. */
  public static final String BLOCK_REWARD_KEY = "blockreward";
 
@@ -82,6 +86,15 @@ public OptionalInt getBlockPeriodSeconds() {
  return JsonUtil.getPositiveInt(forkConfigRoot, BLOCK_PERIOD_SECONDS_KEY);
  }
 
+ /**
+ * Gets block period milliseconds. Experimental for test scenarios only.
+ *
+ * @return the block period milliseconds
+ */
+ public OptionalLong getBlockPeriodMilliseconds() {
+ return JsonUtil.getLong(forkConfigRoot, BLOCK_PERIOD_MILLISECONDS_KEY);
+ }
+
  /**
  * Gets block reward wei.
  *

config/src/main/java/org/hyperledger/besu/config/JsonBftConfigOptions.java

@@ -34,6 +34,7 @@ public class JsonBftConfigOptions implements BftConfigOptions {
 
  private static final long DEFAULT_EPOCH_LENGTH = 30_000;
  private static final int DEFAULT_BLOCK_PERIOD_SECONDS = 1;
+ private static final int DEFAULT_BLOCK_PERIOD_MILLISECONDS = 0; // Experimental for test only
  private static final int DEFAULT_ROUND_EXPIRY_SECONDS = 1;
  // In a healthy network this can be very small. This default limit will allow for suitable
  // protection for on a typical 20 node validator network with multiple rounds
@@ -66,6 +67,12 @@ public int getBlockPeriodSeconds() {
  bftConfigRoot, "blockperiodseconds", DEFAULT_BLOCK_PERIOD_SECONDS);
  }
 
+ @Override
+ public long getBlockPeriodMilliseconds() {
+ return JsonUtil.getLong(
+ bftConfigRoot, "xblockperiodmilliseconds", DEFAULT_BLOCK_PERIOD_MILLISECONDS);
+ }
+
  @Override
  public int getRequestTimeoutSeconds() {
  return JsonUtil.getInt(bftConfigRoot, "requesttimeoutseconds", DEFAULT_ROUND_EXPIRY_SECONDS);
@@ -133,6 +140,9 @@ public Map<String, Object> asMap() {
  if (bftConfigRoot.has("blockperiodseconds")) {
  builder.put("blockPeriodSeconds", getBlockPeriodSeconds());
  }
+ if (bftConfigRoot.has("xblockperiodmilliseconds")) {
+ builder.put("xBlockPeriodMilliSeconds", getBlockPeriodMilliseconds());
+ }
  if (bftConfigRoot.has("requesttimeoutseconds")) {
  builder.put("requestTimeoutSeconds", getRequestTimeoutSeconds());
  }

config/src/main/resources/holesky.json

@@ -21,7 +21,8 @@
  "bootnodes": [
  "enode://ac906289e4b7f12df423d654c5a962b6ebe5b3a74cc9e06292a85221f9a64a6f1cfdd6b714ed6dacef51578f92b34c60ee91e9ede9c7f8fadc4d347326d95e2b@146.190.13.128:30303",
  "enode://a3435a0155a3e837c02f5e7f5662a2f1fbc25b48e4dc232016e1c51b544cb5b4510ef633ea3278c0e970fa8ad8141e2d4d0f9f95456c537ff05fdf9b31c15072@178.128.136.233:30303",
- "enode://7fa09f1e8bb179ab5e73f45d3a7169a946e7b3de5ef5cea3a0d4546677e4435ee38baea4dd10b3ddfdc1f1c5e869052932af8b8aeb6f9738598ec4590d0b11a6@65.109.94.124:30303"
+ "enode://7fa09f1e8bb179ab5e73f45d3a7169a946e7b3de5ef5cea3a0d4546677e4435ee38baea4dd10b3ddfdc1f1c5e869052932af8b8aeb6f9738598ec4590d0b11a6@65.109.94.124:30303",
+ "enode://3524632a412f42dee4b9cc899b946912359bb20103d7596bddb9c8009e7683b7bff39ea20040b7ab64d23105d4eac932d86b930a605e632357504df800dba100@172.174.35.249:30303"
  ]
  }
  },

consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BlockTimer.java

@@ -24,9 +24,14 @@
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /** Class for starting and keeping organised block timers */
 public class BlockTimer {
 
+ private static final Logger LOG = LoggerFactory.getLogger(BlockTimer.class);
+
  private final ForksSchedule<? extends BftConfigOptions> forksSchedule;
  private final BftExecutors bftExecutors;
  private Optional<ScheduledFuture<?>> currentTimerTask;
@@ -79,12 +84,26 @@ public synchronized void startTimer(
  cancelTimer();
 
  final long now = clock.millis();
+ final long expiryTime;
+
+ // Experimental option for test scenarios only. Not for production use.
+ final long blockPeriodMilliseconds =
+ forksSchedule.getFork(round.getSequenceNumber()).getValue().getBlockPeriodMilliseconds();
 
- // absolute time when the timer is supposed to expire
- final int blockPeriodSeconds =
- forksSchedule.getFork(round.getSequenceNumber()).getValue().getBlockPeriodSeconds();
- final long minimumTimeBetweenBlocksMillis = blockPeriodSeconds * 1000L;
- final long expiryTime = chainHeadHeader.getTimestamp() * 1_000 + minimumTimeBetweenBlocksMillis;
+ if (blockPeriodMilliseconds > 0) {
+ // Experimental mode for setting < 1 second block periods e.g. for CI/CD pipelines
+ // running tests against Besu
+ expiryTime = clock.millis() + blockPeriodMilliseconds;
+ LOG.warn(
+ "Test-mode only xblockperiodmilliseconds has been set to {} millisecond blocks. Do not use in a production system.",
+ blockPeriodMilliseconds);
+ } else {
+ // absolute time when the timer is supposed to expire
+ final int blockPeriodSeconds =
+ forksSchedule.getFork(round.getSequenceNumber()).getValue().getBlockPeriodSeconds();
+ final long minimumTimeBetweenBlocksMillis = blockPeriodSeconds * 1000L;
+ expiryTime = chainHeadHeader.getTimestamp() * 1_000 + minimumTimeBetweenBlocksMillis;
+ }
 
  if (expiryTime > now) {
  final long delay = expiryTime - now;

consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/MutableBftConfigOptions.java

@@ -31,6 +31,7 @@
 public class MutableBftConfigOptions implements BftConfigOptions {
  private long epochLength;
  private int blockPeriodSeconds;
+ private long blockPeriodMilliseconds;
  private int requestTimeoutSeconds;
  private int gossipedHistoryLimit;
  private int messageQueueLimit;
@@ -48,6 +49,7 @@ public class MutableBftConfigOptions implements BftConfigOptions {
  public MutableBftConfigOptions(final BftConfigOptions bftConfigOptions) {
  this.epochLength = bftConfigOptions.getEpochLength();
  this.blockPeriodSeconds = bftConfigOptions.getBlockPeriodSeconds();
+ this.blockPeriodMilliseconds = bftConfigOptions.getBlockPeriodMilliseconds();
  this.requestTimeoutSeconds = bftConfigOptions.getRequestTimeoutSeconds();
  this.gossipedHistoryLimit = bftConfigOptions.getGossipedHistoryLimit();
  this.messageQueueLimit = bftConfigOptions.getMessageQueueLimit();
@@ -68,6 +70,11 @@ public int getBlockPeriodSeconds() {
  return blockPeriodSeconds;
  }
 
+ @Override
+ public long getBlockPeriodMilliseconds() {
+ return blockPeriodMilliseconds;
+ }
+
  @Override
  public int getRequestTimeoutSeconds() {
  return requestTimeoutSeconds;
@@ -131,6 +138,16 @@ public void setBlockPeriodSeconds(final int blockPeriodSeconds) {
  this.blockPeriodSeconds = blockPeriodSeconds;
  }
 
+ /**
+ * Sets block period milliseconds. Experimental for test scenarios. Not for use on production
+ * systems.
+ *
+ * @param blockPeriodMilliseconds the block period milliseconds
+ */
+ public void setBlockPeriodMilliseconds(final long blockPeriodMilliseconds) {
+ this.blockPeriodMilliseconds = blockPeriodMilliseconds;
+ }
+
  /**
  * Sets request timeout seconds.
  *

consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/RoundTimer.java

@@ -16,6 +16,7 @@
 
 import org.hyperledger.besu.consensus.common.bft.events.RoundExpiry;
 
+import java.time.Duration;
 import java.util.Optional;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -31,21 +32,21 @@ public class RoundTimer {
  private final BftExecutors bftExecutors;
  private Optional<ScheduledFuture<?>> currentTimerTask;
  private final BftEventQueue queue;
- private final long baseExpiryMillis;
+ private final Duration baseExpiryPeriod;
 
  /**
  * Construct a RoundTimer with primed executor service ready to start timers
  *
  * @param queue The queue in which to put round expiry events
- * @param baseExpirySeconds The initial round length for round 0
+ * @param baseExpiryPeriod The initial round length for round 0
  * @param bftExecutors executor service that timers can be scheduled with
  */
  public RoundTimer(
- final BftEventQueue queue, final long baseExpirySeconds, final BftExecutors bftExecutors) {
+ final BftEventQueue queue, final Duration baseExpiryPeriod, final BftExecutors bftExecutors) {
  this.queue = queue;
  this.bftExecutors = bftExecutors;
  this.currentTimerTask = Optional.empty();
- this.baseExpiryMillis = baseExpirySeconds * 1000;
+ this.baseExpiryPeriod = baseExpiryPeriod;
  }
 
  /** Cancels the current running round timer if there is one */
@@ -71,7 +72,8 @@ public synchronized boolean isRunning() {
  public synchronized void startTimer(final ConsensusRoundIdentifier round) {
  cancelTimer();
 
- final long expiryTime = baseExpiryMillis * (long) Math.pow(2, round.getRoundNumber());
+ final long expiryTime =
+ baseExpiryPeriod.toMillis() * (long) Math.pow(2, round.getRoundNumber());
 
  final Runnable newTimerRunnable = () -> queue.add(new RoundExpiry(round));
 
@@ -81,7 +83,7 @@ public synchronized void startTimer(final ConsensusRoundIdentifier round) {
  // Once we are up to round 2 start logging round expiries
  if (round.getRoundNumber() >= 2) {
  LOG.info(
- "QBFT round {} expired. Moved to round {} which will expire in {} seconds",
+ "BFT round {} expired. Moved to round {} which will expire in {} seconds",
  round.getRoundNumber() - 1,
  round.getRoundNumber(),
  (expiryTime / 1000));

consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/RoundTimerTest.java

@@ -25,6 +25,7 @@
 
 import org.hyperledger.besu.consensus.common.bft.events.RoundExpiry;
 
+import java.time.Duration;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
@@ -46,7 +47,7 @@ public void initialise() {
  bftExecutors = mock(BftExecutors.class);
  queue = new BftEventQueue(1000);
  queue.start();
- timer = new RoundTimer(queue, 1, bftExecutors);
+ timer = new RoundTimer(queue, Duration.ofSeconds(1), bftExecutors);
  }
 
  @Test

consensus/ibft/src/integration-test/java/org/hyperledger/besu/consensus/ibft/support/TestContextBuilder.java

@@ -100,6 +100,7 @@
 import org.hyperledger.besu.util.Subscribers;
 
 import java.time.Clock;
+import java.time.Duration;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.util.ArrayList;
@@ -403,7 +404,7 @@ private static ControllerAndState createControllerAndFinalState(
  Util.publicKeyToAddress(nodeKey.getPublicKey()),
  proposerSelector,
  multicaster,
- new RoundTimer(bftEventQueue, ROUND_TIMER_SEC, bftExecutors),
+ new RoundTimer(bftEventQueue, Duration.ofSeconds(ROUND_TIMER_SEC), bftExecutors),
  new BlockTimer(bftEventQueue, forksSchedule, bftExecutors, TestClock.fixed()),
  blockCreatorFactory,
  clock);