From 7af03b7295cc208d9f3fe9cd2872d49f87f3920a Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 1 Oct 2024 18:00:20 +0200 Subject: [PATCH 1/8] Align gas cap for transaction simulation to Geth approach (#7703) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + .../transaction/TransactionSimulator.java | 47 ++++++++--- .../transaction/TransactionSimulatorTest.java | 84 ++++++++++++++----- 3 files changed, 99 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dce16b3f9d6..02b499b1787 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - LUKSO Cancun Hardfork [#7686](https://github.com/hyperledger/besu/pull/7686) - Add configuration of Consolidation Request Contract Address via genesis configuration [#7647](https://github.com/hyperledger/besu/pull/7647) - Interrupt pending transaction processing on block creation timeout [#7673](https://github.com/hyperledger/besu/pull/7673) +- Align gas cap calculation for transaction simulation to Geth approach [#7703](https://github.com/hyperledger/besu/pull/7703) ### Bug fixes - Fix mounted data path directory permissions for besu user [#7575](https://github.com/hyperledger/besu/pull/7575) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java index 29459ba3da4..5e78256a789 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java @@ -230,16 +230,9 @@ public Optional processWithWorldUpdater( final Account sender = updater.get(senderAddress); final long nonce = sender != null ? sender.getNonce() : 0L; - long gasLimit = - callParams.getGasLimit() >= 0 - ? callParams.getGasLimit() - : blockHeaderToProcess.getGasLimit(); - if (rpcGasCap > 0) { - gasLimit = rpcGasCap; - LOG.trace( - "Gas limit capped at {} for transaction simulation due to provided RPC gas cap.", - rpcGasCap); - } + final long simulationGasCap = + calculateSimulationGasCap(callParams.getGasLimit(), blockHeaderToProcess.getGasLimit()); + final Wei value = callParams.getValue() != null ? callParams.getValue() : Wei.ZERO; final Bytes payload = callParams.getPayload() != null ? callParams.getPayload() : Bytes.EMPTY; @@ -265,7 +258,7 @@ public Optional processWithWorldUpdater( header, senderAddress, nonce, - gasLimit, + simulationGasCap, value, payload, blobGasPrice); @@ -291,6 +284,38 @@ public Optional processWithWorldUpdater( return Optional.of(new TransactionSimulatorResult(transaction, result)); } + private long calculateSimulationGasCap( + final long userProvidedGasLimit, final long blockGasLimit) { + final long simulationGasCap; + + // when not set gas limit is -1 + if (userProvidedGasLimit >= 0) { + if (rpcGasCap > 0 && userProvidedGasLimit > rpcGasCap) { + LOG.trace( + "User provided gas limit {} is bigger than the value of rpc-gas-cap {}, setting simulation gas cap to the latter", + userProvidedGasLimit, + rpcGasCap); + simulationGasCap = rpcGasCap; + } else { + LOG.trace("Using provided gas limit {} set as simulation gas cap", userProvidedGasLimit); + simulationGasCap = userProvidedGasLimit; + } + } else { + if (rpcGasCap > 0) { + LOG.trace( + "No user provided gas limit, setting simulation gas cap to the value of rpc-gas-cap {}", + rpcGasCap); + simulationGasCap = rpcGasCap; + } else { + simulationGasCap = blockGasLimit; + LOG.trace( + "No user provided gas limit and rpc-gas-cap options is not set, setting simulation gas cap to block gas limit {}", + blockGasLimit); + } + } + return simulationGasCap; + } + private Optional buildTransaction( final CallParameter callParams, final TransactionValidationParams transactionValidationParams, diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java index 0aac2f026f7..73fa402abbd 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java @@ -79,7 +79,8 @@ public class TransactionSimulatorTest { private static final Address DEFAULT_FROM = Address.fromHexString("0x0000000000000000000000000000000000000000"); - private static final long GASCAP = 500L; + private static final long GAS_CAP = 500000L; + private static final long TRANSFER_GAS_LIMIT = 21000L; private TransactionSimulator transactionSimulator; private TransactionSimulator cappedTransactionSimulator; @@ -96,7 +97,7 @@ public void setUp() { this.transactionSimulator = new TransactionSimulator(blockchain, worldStateArchive, protocolSchedule, 0); this.cappedTransactionSimulator = - new TransactionSimulator(blockchain, worldStateArchive, protocolSchedule, GASCAP); + new TransactionSimulator(blockchain, worldStateArchive, protocolSchedule, GAS_CAP); } @Test @@ -124,7 +125,7 @@ public void shouldReturnSuccessfulResultWhenProcessingIsSuccessful() { .type(TransactionType.FRONTIER) .nonce(1L) .gasPrice(callParameter.getGasPrice()) - .gasLimit(callParameter.getGasLimit()) + .gasLimit(blockHeader.getGasLimit()) .to(callParameter.getTo()) .sender(callParameter.getFrom()) .value(callParameter.getValue()) @@ -155,7 +156,7 @@ public void shouldSetGasPriceToZeroWhenExceedingBalanceAllowed() { .type(TransactionType.FRONTIER) .nonce(1L) .gasPrice(Wei.ZERO) - .gasLimit(callParameter.getGasLimit()) + .gasLimit(blockHeader.getGasLimit()) .to(callParameter.getTo()) .sender(callParameter.getFrom()) .value(callParameter.getValue()) @@ -175,7 +176,8 @@ public void shouldSetGasPriceToZeroWhenExceedingBalanceAllowed() { @Test public void shouldSetFeePerGasToZeroWhenExceedingBalanceAllowed() { - final CallParameter callParameter = eip1559TransactionCallParameter(Wei.ONE, Wei.ONE); + final CallParameter callParameter = + eip1559TransactionCallParameter(Wei.ONE, Wei.ONE, TRANSFER_GAS_LIMIT); final BlockHeader blockHeader = mockBlockHeader(Hash.ZERO, 1L, Wei.ONE); @@ -187,7 +189,7 @@ public void shouldSetFeePerGasToZeroWhenExceedingBalanceAllowed() { .type(TransactionType.EIP1559) .chainId(BigInteger.ONE) .nonce(1L) - .gasLimit(callParameter.getGasLimit()) + .gasLimit(TRANSFER_GAS_LIMIT) .maxFeePerGas(Wei.ZERO) .maxPriorityFeePerGas(Wei.ZERO) .to(callParameter.getTo()) @@ -223,7 +225,7 @@ public void shouldNotSetGasPriceToZeroWhenExceedingBalanceIsNotAllowed() { .type(TransactionType.FRONTIER) .nonce(1L) .gasPrice(callParameter.getGasPrice()) - .gasLimit(callParameter.getGasLimit()) + .gasLimit(blockHeader.getGasLimit()) .to(callParameter.getTo()) .sender(callParameter.getFrom()) .value(callParameter.getValue()) @@ -244,7 +246,8 @@ public void shouldNotSetGasPriceToZeroWhenExceedingBalanceIsNotAllowed() { @Test public void shouldNotSetFeePerGasToZeroWhenExceedingBalanceIsNotAllowed() { - final CallParameter callParameter = eip1559TransactionCallParameter(Wei.ONE, Wei.ONE); + final CallParameter callParameter = + eip1559TransactionCallParameter(Wei.ONE, Wei.ONE, TRANSFER_GAS_LIMIT); final BlockHeader blockHeader = mockBlockHeader(Hash.ZERO, 1L, Wei.ONE); @@ -256,7 +259,7 @@ public void shouldNotSetFeePerGasToZeroWhenExceedingBalanceIsNotAllowed() { .type(TransactionType.EIP1559) .chainId(BigInteger.ONE) .nonce(1L) - .gasLimit(callParameter.getGasLimit()) + .gasLimit(TRANSFER_GAS_LIMIT) .maxFeePerGas(callParameter.getMaxFeePerGas().orElseThrow()) .maxPriorityFeePerGas(callParameter.getMaxPriorityFeePerGas().orElseThrow()) .to(callParameter.getTo()) @@ -349,7 +352,7 @@ public void shouldReturnFailureResultWhenProcessingFails() { .type(TransactionType.FRONTIER) .nonce(1L) .gasPrice(callParameter.getGasPrice()) - .gasLimit(callParameter.getGasLimit()) + .gasLimit(blockHeader.getGasLimit()) .to(callParameter.getTo()) .sender(callParameter.getFrom()) .value(callParameter.getValue()) @@ -390,7 +393,7 @@ public void shouldReturnSuccessfulResultWhenProcessingIsSuccessfulByHash() { .type(TransactionType.FRONTIER) .nonce(1L) .gasPrice(callParameter.getGasPrice()) - .gasLimit(callParameter.getGasLimit()) + .gasLimit(blockHeader.getGasLimit()) .to(callParameter.getTo()) .sender(callParameter.getFrom()) .value(callParameter.getValue()) @@ -479,7 +482,7 @@ public void shouldReturnFailureResultWhenProcessingFailsByHash() { .type(TransactionType.FRONTIER) .nonce(1L) .gasPrice(callParameter.getGasPrice()) - .gasLimit(callParameter.getGasLimit()) + .gasLimit(blockHeader.getGasLimit()) .to(callParameter.getTo()) .sender(callParameter.getFrom()) .value(callParameter.getValue()) @@ -509,7 +512,7 @@ public void shouldReturnSuccessfulResultWhenEip1559TransactionProcessingIsSucces .type(TransactionType.EIP1559) .chainId(BigInteger.ONE) .nonce(1L) - .gasLimit(callParameter.getGasLimit()) + .gasLimit(blockHeader.getGasLimit()) .maxFeePerGas(callParameter.getMaxFeePerGas().orElseThrow()) .maxPriorityFeePerGas(callParameter.getMaxPriorityFeePerGas().orElseThrow()) .to(callParameter.getTo()) @@ -530,7 +533,7 @@ public void shouldReturnSuccessfulResultWhenEip1559TransactionProcessingIsSucces @Test public void shouldCapGasLimitWhenOriginalTransactionExceedsGasCap() { final CallParameter callParameter = - eip1559TransactionCallParameter(Wei.ZERO, Wei.ZERO, GASCAP + 1); + eip1559TransactionCallParameter(Wei.ZERO, Wei.ZERO, GAS_CAP + 1); final BlockHeader blockHeader = mockBlockHeader(Hash.ZERO, 1L, Wei.ONE); @@ -542,7 +545,7 @@ public void shouldCapGasLimitWhenOriginalTransactionExceedsGasCap() { .type(TransactionType.EIP1559) .chainId(BigInteger.ONE) .nonce(1L) - .gasLimit(GASCAP) + .gasLimit(GAS_CAP) .maxFeePerGas(callParameter.getMaxFeePerGas().orElseThrow()) .maxPriorityFeePerGas(callParameter.getMaxPriorityFeePerGas().orElseThrow()) .to(callParameter.getTo()) @@ -566,11 +569,48 @@ public void shouldCapGasLimitWhenOriginalTransactionExceedsGasCap() { } @Test - public void shouldUseRpcGasCapWhenCapIsHigherThanGasLimit() { - // generate a transaction with a gas limit that is lower than the gas cap, - // expect the gas cap to override parameter gas limit + public void shouldUseProvidedGasLimitWhenBelowRpcCapGas() { final CallParameter callParameter = - eip1559TransactionCallParameter(Wei.ZERO, Wei.ZERO, GASCAP - 1); + eip1559TransactionCallParameter(Wei.ZERO, Wei.ZERO, GAS_CAP / 2); + + final BlockHeader blockHeader = mockBlockHeader(Hash.ZERO, 1L, Wei.ONE); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, callParameter.getFrom(), 1L); + + final Transaction expectedTransaction = + Transaction.builder() + .type(TransactionType.EIP1559) + .chainId(BigInteger.ONE) + .nonce(1L) + .gasLimit(GAS_CAP / 2) + .maxFeePerGas(callParameter.getMaxFeePerGas().orElseThrow()) + .maxPriorityFeePerGas(callParameter.getMaxPriorityFeePerGas().orElseThrow()) + .to(callParameter.getTo()) + .sender(callParameter.getFrom()) + .value(callParameter.getValue()) + .payload(callParameter.getPayload()) + .signature(FAKE_SIGNATURE) + .build(); + + mockProtocolSpecForProcessWithWorldUpdater(); + + // call process with original transaction + cappedTransactionSimulator.process( + callParameter, + TransactionValidationParams.transactionSimulator(), + OperationTracer.NO_TRACING, + 1L); + + // expect overwritten transaction to be processed + verifyTransactionWasProcessed(expectedTransaction); + } + + @Test + public void shouldUseRpcGasCapWhenGasLimitNoPresent() { + // generate call parameters that do not specify a gas limit, + // expect the rpc gas cap to be used for simulation + final CallParameter callParameter = eip1559TransactionCallParameter(Wei.ZERO, Wei.ZERO, -1); final BlockHeader blockHeader = mockBlockHeader(Hash.ZERO, 1L, Wei.ONE); @@ -591,7 +631,7 @@ public void shouldUseRpcGasCapWhenCapIsHigherThanGasLimit() { .value(callParameter.getValue()) .payload(callParameter.getPayload()) .signature(FAKE_SIGNATURE) - .gasLimit(GASCAP) + .gasLimit(GAS_CAP) .build(); // call process with original transaction @@ -781,7 +821,7 @@ private CallParameter legacyTransactionCallParameter(final Wei gasPrice) { return new CallParameter( Address.fromHexString("0x0"), Address.fromHexString("0x0"), - 0, + -1, gasPrice, Wei.of(0), Bytes.EMPTY); @@ -793,7 +833,7 @@ private CallParameter eip1559TransactionCallParameter() { private CallParameter eip1559TransactionCallParameter( final Wei maxFeePerGas, final Wei maxPriorityFeePerGas) { - return eip1559TransactionCallParameter(maxFeePerGas, maxPriorityFeePerGas, 0L); + return eip1559TransactionCallParameter(maxFeePerGas, maxPriorityFeePerGas, -1); } private CallParameter eip1559TransactionCallParameter( From 6c47cc9d4e8edf5df12d70ef953d9e25574c0994 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Tue, 1 Oct 2024 15:10:44 -0700 Subject: [PATCH 2/8] add Luis as a maintainer (#7603) Signed-off-by: garyschulte Co-authored-by: Sally MacFarlane --- MAINTAINERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 17958bdbd43..6b699a6dca6 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -22,6 +22,7 @@ | Justin Florentine| jflo | RoboCopsGoneMad | | Jason Frame | jframe | jframe | | Joshua Fernandes | joshuafernandes | joshuafernandes | +| Luis Pinto | lu-pinto | lu-pinto | Lucas Saldanha | lucassaldanha | lucassaldanha | | Sally MacFarlane | macfarla | macfarla | | Karim Taam | matkt | matkt | From 5f6f7a077a7ace10743aba29c9a3df3a3bee4946 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Wed, 2 Oct 2024 09:05:11 +1000 Subject: [PATCH 3/8] Moved maintainers to emeritus (#7611) * move antoine to emeritus Signed-off-by: Sally MacFarlane * moved jiri and antony to emeritus Signed-off-by: Sally MacFarlane * add adrian Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane Co-authored-by: Usman Saleem --- MAINTAINERS.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 6b699a6dca6..cf040c25bc0 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -9,15 +9,11 @@ | Name | Github | LFID | | ---------------- | ---------------- | ---------------- | | Ameziane Hamlat | ahamlat | ahamlat | -| Adrian Sutton | ajsutton | ajsutton | -| Antony Denyer | antonydenyer | antonydenyer | -| Antoine Toulme | atoulme | atoulme | | Daniel Lehrner | daniellehrner | daniellehrner | | Diego López León | diega | diega | | Fabio Di Fabio | fab-10 | fab-10 | | Gabriel Trintinalia | gabriel-trintinalia | gabrieltrintinalia | | Gary Schulte | garyschulte | GarySchulte | -| Jiri Peinlich | gezero | JiriPeinlich | | Gabriel Fukushima| gfukushima | gfukushima | | Justin Florentine| jflo | RoboCopsGoneMad | | Jason Frame | jframe | jframe | @@ -38,11 +34,15 @@ | Name | Github | LFID | |------------------|------------------|------------------| | Abdel Bakhta | abdelhamidbakhta | abdelhamidbakhta | +| Adrian Sutton | ajsutton | ajsutton | +| Antony Denyer | antonydenyer | antonydenyer | +| Antoine Toulme | atoulme | atoulme | | Byron Gravenorst | bgravenorst | bgravenorst | | Chris Hare | CjHare | cjhare | | David Mechler | davemec | davemec | | Edward Evans | EdJoJob | EdJoJob | | Edward Mack | edwardmack | mackcom | +| Jiri Peinlich | gezero | JiriPeinlich | | Frank Li | frankisawesome | frankliawesome | | Ivaylo Kirilov | iikirilov | iikirilov | | Madeline Murray | MadelineMurray | madelinemurray | From db1899c227f04b8beb92a1de9603325d9716ecc2 Mon Sep 17 00:00:00 2001 From: Bhanu Pulluri <59369753+pullurib@users.noreply.github.com> Date: Tue, 1 Oct 2024 19:45:34 -0400 Subject: [PATCH 4/8] Expose revert reason field through GraphQL (#7677) Signed-off-by: Bhanu Pulluri Co-authored-by: Bhanu Pulluri Co-authored-by: Sally MacFarlane --- .../pojoadapter/TransactionAdapter.java | 18 ++++++++++++++++++ .../api/src/main/resources/schema.graphqls | 3 +++ 2 files changed, 21 insertions(+) diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java index d5eacf8e357..f8e60c5527c 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.LogWithMetadata; import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; @@ -292,6 +293,23 @@ public Optional getStatus(final DataFetchingEnvironment environment) { : Optional.of((long) receipt.getStatus())); } + /** + * Retrieves the revert reason of the transaction, if any. + * + *

This method uses the getReceipt method to get the receipt of the transaction. It then checks + * the revert reason of the receipt. It would be an empty Optional for successful transactions. + * Otherwise, it returns an Optional containing the revert reason. + * + * @param environment the data fetching environment. + * @return an Optional containing a Bytes object representing the revert reason of the + * transaction, or an empty Optional . + */ + public Optional getRevertReason(final DataFetchingEnvironment environment) { + return getReceipt(environment) + .map(TransactionReceiptWithMetadata::getReceipt) + .flatMap(TransactionReceipt::getRevertReason); + } + /** * Retrieves the gas used by the transaction. * diff --git a/ethereum/api/src/main/resources/schema.graphqls b/ethereum/api/src/main/resources/schema.graphqls index 87c7509b393..19d3ffea460 100644 --- a/ethereum/api/src/main/resources/schema.graphqls +++ b/ethereum/api/src/main/resources/schema.graphqls @@ -579,6 +579,9 @@ type Transaction { BlobVersionedHashes is a set of hash outputs from the blobs in the transaction. """ blobVersionedHashes: [Bytes32!] + + """Reason returned when transaction fails.""" + revertReason: Bytes } """EIP-4895""" From fbec990bd2583752026b1f70a9f45e69685afe80 Mon Sep 17 00:00:00 2001 From: Chaminda Divitotawela Date: Wed, 2 Oct 2024 10:53:03 +1000 Subject: [PATCH 5/8] Update release process (#7707) Previously GitHub release will be created which triggers building and publishing the artifacts. Release engineers found workflow could fail after the GitHub release created. Community may subscribed to GitHub releases but if the workflow failed, artifacts for the release would not available. Proposed solution requires release engineer to run a GitHub workflow manually by providing the Git tag which creates a draft GitHub release. During this workflow, release artifacts binary distribution, docker images (not latest), Artifactory jars created and published. Release engineer can update the release notes in draft release and publish the release. At the time when the release is published, release engineer is confident all the artifacts are ready. Upon publishing the release, another workflow is triggered to promote the release version of docker images to latest Signed-off-by: Chaminda Divitotawela Co-authored-by: Simon Dudley --- .github/workflows/docker-promote.yml | 81 +++++ .../{release.yml => draft-release.yml} | 314 +++++++++++------- 2 files changed, 267 insertions(+), 128 deletions(-) create mode 100644 .github/workflows/docker-promote.yml rename .github/workflows/{release.yml => draft-release.yml} (59%) diff --git a/.github/workflows/docker-promote.yml b/.github/workflows/docker-promote.yml new file mode 100644 index 00000000000..858ea7d20e8 --- /dev/null +++ b/.github/workflows/docker-promote.yml @@ -0,0 +1,81 @@ +name: Docker Promote + +run-name: "Docker Promote ${{ github.event.release.name }}" + +on: + release: + types: [released] + +env: + registry: docker.io + GRADLE_OPTS: "-Dorg.gradle.parallel=true -Dorg.gradle.caching=true" + +jobs: + validate: + runs-on: ubuntu-22.04 + env: + RELEASE_NAME: "${{ github.event.release.name }}" + steps: + - name: Pre-process Release Name + id: pre_process_release_name + run: | + # strip all whitespace + RELEASE_NAME="${RELEASE_NAME//[[:space:]]/}" + if [[ ! "$RELEASE_NAME" =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?(-.*)?$ ]]; then + echo "Release name does not conform to a valid besu release format YY.M.v[-suffix], e.g. 24.8.0-RC1." + exit 1 + fi + echo "release_name=$RELEASE_NAME" >> $GITHUB_OUTPUT # Set as output using the new syntax + outputs: + release_name: ${{ steps.pre_process_release_name.outputs.release_name }} + + docker-promote: + needs: [validate] + env: + RELEASE_NAME: ${{ needs.validate.outputs.release_name }} # Use the output from the pre_process_release job + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + + - name: Setup Java + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 + with: + distribution: temurin + java-version: 21 + cache: gradle + + - name: Login to ${{ env.registry }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d + with: + registry: ${{ env.registry }} + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true + + - name: Docker upload + run: ./gradlew "-Prelease.releaseVersion=${{ env.RELEASE_NAME }}" "-PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }}" dockerUploadRelease + + - name: Docker manifest + run: ./gradlew "-Prelease.releaseVersion=${{ env.RELEASE_NAME }}" "-PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }}" manifestDockerRelease + + docker-verify: + needs: [validate, docker-promote] + env: + RELEASE_NAME: ${{ needs.validate.outputs.release_name }} # Use the output from the pre_process_release job + runs-on: ubuntu-22.04 + permissions: + contents: read + actions: write + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + + - name: Trigger container verify + run: echo '{"version":"${{ env.RELEASE_NAME }}","verify-latest-version":"true"}' | gh workflow run container-verify.yml --json + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/release.yml b/.github/workflows/draft-release.yml similarity index 59% rename from .github/workflows/release.yml rename to .github/workflows/draft-release.yml index 7c9b70891d1..f6e01e1890e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/draft-release.yml @@ -1,21 +1,35 @@ -name: release +name: Draft Release + +run-name: "Draft Release ${{ inputs.tag }}" on: - release: - types: [released] + workflow_dispatch: + inputs: + tag: + required: true env: registry: docker.io GRADLE_OPTS: "-Dorg.gradle.parallel=true -Dorg.gradle.caching=true" jobs: - preprocess_release: + validate: runs-on: ubuntu-22.04 + env: + RELEASE_NAME: "${{ inputs.tag }}" steps: + - name: Check default branch + run: | + echo "Current Branch: ${{ github.ref_name }}" + echo "Default Branch: ${{ github.event.repository.default_branch }}" + if [[ ${{ github.ref_name }} != ${{ github.event.repository.default_branch }} ]] + then + echo "This workflow can only be run on default branch" + exit 1 + fi + - name: Pre-process Release Name id: pre_process_release_name - env: - RELEASE_NAME: "${{ github.event.release.name }}" run: | # strip all whitespace RELEASE_NAME="${RELEASE_NAME//[[:space:]]/}" @@ -24,35 +38,47 @@ jobs: exit 1 fi echo "release_name=$RELEASE_NAME" >> $GITHUB_OUTPUT # Set as output using the new syntax + + # Perform a tag checkout to ensure tag is available + - name: Verify tag Exist + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ steps.pre_process_release_name.outputs.release_name }} + fetch-depth: 1 + outputs: release_name: ${{ steps.pre_process_release_name.outputs.release_name }} - artifacts: + build: runs-on: ubuntu-22.04 - needs: preprocess_release + needs: validate env: - RELEASE_NAME: ${{ needs.preprocess_release.outputs.release_name }} # Use the output from the pre_process_release job - permissions: - contents: write + RELEASE_NAME: ${{ needs.validate.outputs.release_name }} # Use the output from the pre_process_release job outputs: tarSha: ${{steps.hashes.outputs.tarSha}} zipSha: ${{steps.hashes.outputs.zipSha}} steps: - - name: checkout + - name: Checkout tag uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ env.RELEASE_NAME }} + - name: Set up Java uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 21 - - name: setup gradle + + - name: Setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 with: cache-disabled: true - - name: assemble release + + - name: Assemble release run: ./gradlew -Prelease.releaseVersion=${{env.RELEASE_NAME}} -Pversion=${{env.RELEASE_NAME}} assemble - - name: hashes + + - name: Hashes id: hashes run: | cd build/distributions @@ -60,37 +86,56 @@ jobs: echo "tarSha=$(shasum -a 256 besu*.tar.gz)" echo "zipSha=$(shasum -a 256 besu*.zip)" >> $GITHUB_OUTPUT echo "tarSha=$(shasum -a 256 besu*.tar.gz)" >> $GITHUB_OUTPUT - - name: upload tarball + shasum -a 256 besu-${{env.RELEASE_NAME}}.tar.gz > besu-${{env.RELEASE_NAME}}.tar.gz.sha256 + shasum -a 256 besu-${{env.RELEASE_NAME}}.zip > besu-${{env.RELEASE_NAME}}.zip.sha256 + + - name: Upload tarball uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 with: - path: 'build/distributions/besu*.tar.gz' + path: 'build/distributions/besu-${{ env.RELEASE_NAME }}.tar.gz' name: besu-${{ env.RELEASE_NAME }}.tar.gz compression-level: 0 + - name: upload zipfile uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 with: - path: 'build/distributions/besu*.zip' + path: 'build/distributions/besu-${{ env.RELEASE_NAME }}.zip' name: besu-${{ env.RELEASE_NAME }}.zip compression-level: 0 - testWindows: + - name: upload checksum zip + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 + with: + path: 'build/distributions/besu-${{ env.RELEASE_NAME }}.zip.sha256' + name: besu-${{ env.RELEASE_NAME }}.zip.sha256 + compression-level: 0 + + - name: upload checksum tar.gz + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 + with: + path: 'build/distributions/besu-${{ env.RELEASE_NAME }}.tar.gz.sha256' + name: besu-${{ env.RELEASE_NAME }}.tar.gz.sha256 + compression-level: 0 + + test-windows: runs-on: windows-2022 - needs: artifacts - timeout-minutes: 10 + needs: ["build"] + timeout-minutes: 5 steps: - name: Set up Java uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 21 + - name: Download zip uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe with: pattern: besu-*.zip merge-multiple: true - - name: test Besu + + - name: Test run: | - dir unzip besu-*.zip -d besu-tmp cd besu-tmp mv besu-* ../besu @@ -98,81 +143,49 @@ jobs: besu\bin\besu.bat --help besu\bin\besu.bat --version - publish: - runs-on: ubuntu-22.04 - needs: [preprocess_release, testWindows, artifacts] - env: - RELEASE_NAME: ${{ needs.preprocess_release.outputs.release_name }} # Use the output from the pre_process_release job - permissions: - contents: write - steps: - - name: Download archives - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe - with: - pattern: besu-* - merge-multiple: true - path: 'build/distributions' - - name: Upload Release assets - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 - with: - append_body: true - files: | - build/distributions/besu*.tar.gz - build/distributions/besu*.zip - body: | - ${{needs.artifacts.outputs.tarSha}} - ${{needs.artifacts.outputs.zipSha}} - - - - artifactoryPublish: + test-linux: runs-on: ubuntu-22.04 - needs: [preprocess_release, artifacts] - env: - RELEASE_NAME: ${{ needs.preprocess_release.outputs.release_name }} # Use the output from the pre_process_release job + needs: ["build"] + timeout-minutes: 5 steps: - - name: checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Set up Java uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 21 - - name: setup gradle - uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + + - name: Download tar.gz + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe with: - cache-disabled: true - - name: Artifactory Publish - env: - ARTIFACTORY_USER: ${{ secrets.BESU_ARTIFACTORY_USER }} - ARTIFACTORY_KEY: ${{ secrets.BESU_ARTIFACTORY_TOKEN }} - run: ./gradlew -Prelease.releaseVersion=${{ env.RELEASE_NAME }} -Pversion=${{env.RELEASE_NAME}} artifactoryPublish + pattern: besu-*.tar.gz + merge-multiple: true + + - name: Test + run: | + tar zxvf besu-*.tar.gz + rm -f besu-*.tar.gz + mv besu-* besu-test + besu-test/bin/besu --help + besu-test/bin/besu --version - hadolint: + docker-lint: runs-on: ubuntu-22.04 + needs: [test-linux, test-windows] + env: + RELEASE_NAME: ${{ needs.validate.outputs.release_name }} # Use the output from the pre_process_release job steps: - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - name: Set up Java - uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: - distribution: temurin - java-version: 21 - - name: setup gradle - uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 - with: - cache-disabled: true + ref: ${{ env.RELEASE_NAME }} + - name: hadoLint run: docker run --rm -i hadolint/hadolint < docker/Dockerfile - - buildDocker: - needs: [preprocess_release, hadolint] + + docker-publish: + needs: [validate, docker-lint] env: - RELEASE_NAME: ${{ needs.preprocess_release.outputs.release_name }} # Use the output from the pre_process_release job - permissions: - contents: read - packages: write - + RELEASE_NAME: ${{ needs.validate.outputs.release_name }} # Use the output from the pre_process_release job strategy: fail-fast: false matrix: @@ -192,30 +205,39 @@ jobs: echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT echo "ARCH=arm64" >> $GITHUB_OUTPUT fi + - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ env.RELEASE_NAME }} + - name: short sha id: shortSha run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - name: Set up Java uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 21 + - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 with: cache-disabled: true + - name: install goss run: | mkdir -p docker/reports curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} + - name: login to ${{ env.registry }} uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d with: registry: ${{ env.registry }} username: ${{ secrets.DOCKER_USER_RW }} password: ${{ secrets.DOCKER_PASSWORD_RW }} + - name: build and test docker uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 env: @@ -223,94 +245,130 @@ jobs: with: cache-disabled: true arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{env.RELEASE_NAME}} -Prelease.releaseVersion=${{ env.RELEASE_NAME }} + - name: publish env: architecture: ${{ steps.prep.outputs.ARCH }} run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{env.RELEASE_NAME}} -Prelease.releaseVersion=${{ env.RELEASE_NAME }} - multiArch: - needs: [preprocess_release, buildDocker] - env: - RELEASE_NAME: ${{ needs.preprocess_release.outputs.release_name }} # Use the output from the pre_process_release job + docker-manifest: + needs: [validate, docker-publish] runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write + env: + RELEASE_NAME: ${{ needs.validate.outputs.release_name }} # Use the output from the pre_process_release job steps: - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ env.RELEASE_NAME }} + - name: Set up Java uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 21 + - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 with: cache-disabled: true + - name: login to ${{ env.registry }} uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d with: registry: ${{ env.registry }} username: ${{ secrets.DOCKER_USER_RW }} password: ${{ secrets.DOCKER_PASSWORD_RW }} + - name: multi-arch docker run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{env.RELEASE_NAME}} -Prelease.releaseVersion=${{ env.RELEASE_NAME }} - amendNotes: - needs: [preprocess_release, multiArch] + docker-verify: + needs: [validate,docker-manifest] env: - RELEASE_NAME: ${{ needs.preprocess_release.outputs.release_name }} # Use the output from the pre_process_release job + RELEASE_NAME: ${{ needs.validate.outputs.release_name }} # Use the output from the pre_process_release job + runs-on: ubuntu-22.04 + permissions: + contents: read + actions: write + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + + - name: Trigger container verify + run: echo '{"version":"${{ env.RELEASE_NAME }}","verify-latest-version":"false"}' | gh workflow run container-verify.yml --json + env: + GH_TOKEN: ${{ github.token }} + + release-draft: runs-on: ubuntu-22.04 + needs: [validate, test-linux, test-windows] permissions: contents: write + env: + RELEASE_NAME: ${{ needs.validate.outputs.release_name }} steps: - - name: add pull command to release notes - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 + - name: Checkout Repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: - append_body: true - body: | - `docker pull ${{env.registry}}/${{secrets.DOCKER_ORG}}/besu:${{env.RELEASE_NAME}}` + ref: ${{ env.RELEASE_NAME }} - dockerPromoteX64: - needs: [preprocess_release, multiArch] - env: - RELEASE_NAME: ${{ needs.preprocess_release.outputs.release_name }} # Use the output from the pre_process_release job + - name: Download Besu artifacts + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe + with: + pattern: besu-${{env.RELEASE_NAME}}* + merge-multiple: true + + - name: Draft release notes + run: | + echo "## ${{env.RELEASE_NAME}}" > draft-release-notes.md + echo "## Upcoming Breaking Changes" >> draft-release-notes.md + echo "## Breaking Changes" >> draft-release-notes.md + echo "## Additions and Improvements" >> draft-release-notes.md + echo "## Bug fixes" >> draft-release-notes.md + echo "`$(cat besu-${{env.RELEASE_NAME}}.zip.sha256)`" >> draft-release-notes.md + echo "`$(cat besu-${{env.RELEASE_NAME}}.tar.gz.sha256)`" >> draft-release-notes.md + cat besu-${{env.RELEASE_NAME}}.zip.sha256 >> draft-release-notes.md + cat besu-${{env.RELEASE_NAME}}.tar.gz.sha256 >> draft-release-notes.md + + - name: Draft release + run: | + gh release create \ + --draft \ + --title=${{env.RELEASE_NAME}} \ + --notes-file draft-release-notes.md \ + --verify-tag ${{env.RELEASE_NAME}} \ + besu-${{env.RELEASE_NAME}}.tar.gz \ + besu-${{env.RELEASE_NAME}}.zip \ + besu-${{env.RELEASE_NAME}}.zip.sha256 \ + besu-${{env.RELEASE_NAME}}.tar.gz.sha256 + env: + GH_TOKEN: ${{ github.token }} + + artifactory: runs-on: ubuntu-22.04 + needs: [validate, test-linux, test-windows] + env: + RELEASE_NAME: ${{ needs.validate.outputs.release_name }} steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 + - name: checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ env.RELEASE_NAME }} + + - name: Set up Java + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 21 - cache: gradle - - name: login to ${{ env.registry }} - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d - with: - registry: ${{ env.registry }} - username: ${{ secrets.DOCKER_USER_RW }} - password: ${{ secrets.DOCKER_PASSWORD_RW }} - - name: Setup Gradle + + - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 with: cache-disabled: true - - name: Docker upload - run: ./gradlew "-Prelease.releaseVersion=${{ env.RELEASE_NAME }}" "-PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }}" dockerUploadRelease - - name: Docker manifest - run: ./gradlew "-Prelease.releaseVersion=${{ env.RELEASE_NAME }}" "-PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }}" manifestDockerRelease - verifyContainer: - needs: [preprocess_release, dockerPromoteX64] - env: - RELEASE_NAME: ${{ needs.preprocess_release.outputs.release_name }} # Use the output from the pre_process_release job - runs-on: ubuntu-22.04 - permissions: - contents: read - actions: write - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - name: Trigger container verify - run: echo '{"version":"${{ env.RELEASE_NAME }}","verify-latest-version":"true"}' | gh workflow run container-verify.yml --json + - name: Artifactory Publish env: - GH_TOKEN: ${{ github.token }} + ARTIFACTORY_USER: ${{ secrets.BESU_ARTIFACTORY_USER }} + ARTIFACTORY_KEY: ${{ secrets.BESU_ARTIFACTORY_TOKEN }} + run: ./gradlew -Prelease.releaseVersion=${{ env.RELEASE_NAME }} -Pversion=${{env.RELEASE_NAME}} artifactoryPublish From 49bf37cc31f22b29e539e205bd362695e7e9aeb6 Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Wed, 2 Oct 2024 12:45:26 +1000 Subject: [PATCH 6/8] Expose chain ID in the BlockchainService plugin API (#7702) Signed-off-by: Gabriel-Trintinalia --- CHANGELOG.md | 1 + .../hyperledger/besu/services/BlockchainServiceImpl.java | 9 +++++++++ plugin-api/build.gradle | 2 +- .../besu/plugin/services/BlockchainService.java | 9 ++++++++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02b499b1787..cb965bfb107 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - Add configuration of Consolidation Request Contract Address via genesis configuration [#7647](https://github.com/hyperledger/besu/pull/7647) - Interrupt pending transaction processing on block creation timeout [#7673](https://github.com/hyperledger/besu/pull/7673) - Align gas cap calculation for transaction simulation to Geth approach [#7703](https://github.com/hyperledger/besu/pull/7703) +- Expose chainId in the `BlockchainService` [7702](https://github.com/hyperledger/besu/pull/7702) ### Bug fixes - Fix mounted data path directory permissions for besu user [#7575](https://github.com/hyperledger/besu/pull/7575) diff --git a/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java b/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java index 1f014ee0610..e981a3e78d6 100644 --- a/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java +++ b/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java @@ -30,6 +30,7 @@ import org.hyperledger.besu.plugin.data.TransactionReceipt; import org.hyperledger.besu.plugin.services.BlockchainService; +import java.math.BigInteger; import java.util.List; import java.util.Optional; import java.util.function.Supplier; @@ -182,4 +183,12 @@ public BlockBody getBlockBody() { } }; } + + @Override + public Optional getChainId() { + if (protocolSchedule == null) { + return Optional.empty(); + } + return protocolSchedule.getChainId(); + } } diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 6d3a1f93a9a..dd2da0a15ec 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -71,7 +71,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = '5H+3gUzCwZtLByfnk11kf+kAPwykQ+WR+n3xWgyfsyY=' + knownHash = '4jVaj9yW88nHbX0KmTR3dPQRvj9x8Pvh5E9Ry7KRT6w=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BlockchainService.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BlockchainService.java index 84a573aadc3..a89d1144af7 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BlockchainService.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BlockchainService.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.plugin.data.BlockHeader; import org.hyperledger.besu.plugin.data.TransactionReceipt; +import java.math.BigInteger; import java.util.List; import java.util.Optional; @@ -106,5 +107,11 @@ void setFinalizedBlock(Hash blockHash) * @throws UnsupportedOperationException if the network is a PoS network */ void setSafeBlock(Hash blockHash) throws IllegalArgumentException, UnsupportedOperationException; - ; + + /** + * Get the chain id + * + * @return the chain id + */ + Optional getChainId(); } From 9310e1031df10107cfb07516fc245798bf4e493d Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Wed, 2 Oct 2024 10:00:52 +0200 Subject: [PATCH 7/8] Fix storage range issue during snapsync (#7624) Signed-off-by: Karim Taam Co-authored-by: Simon Dudley --- .../request/StorageRangeDataRequest.java | 10 +- .../snapsync/AccountHealingTrackingTest.java | 105 +++++++++++++++--- 2 files changed, 94 insertions(+), 21 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java index 14f92e2c3f9..4c50926303b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java @@ -16,7 +16,6 @@ import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.STORAGE_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.StackTrie.FlatDatabaseUpdater.noop; -import static org.hyperledger.besu.ethereum.trie.RangeManager.MAX_RANGE; import static org.hyperledger.besu.ethereum.trie.RangeManager.MIN_RANGE; import static org.hyperledger.besu.ethereum.trie.RangeManager.findNewBeginElementInRange; import static org.hyperledger.besu.ethereum.trie.RangeManager.getRangeCount; @@ -192,12 +191,13 @@ public Stream getChildRequests( getRootHash(), accountHash, storageRoot, key, value); childRequests.add(storageRangeDataRequest); }); - if (startKeyHash.equals(MIN_RANGE) && endKeyHash.equals(MAX_RANGE)) { - // need to heal this account storage - downloadState.addAccountToHealingList(CompactEncoding.bytesToPath(accountHash)); - } }); + if (startKeyHash.equals(MIN_RANGE) && !taskElement.proofs().isEmpty()) { + // need to heal this account storage + downloadState.addAccountToHealingList(CompactEncoding.bytesToPath(accountHash)); + } + return childRequests.stream(); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java index 70bdbf8b801..8e4e40311f3 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java @@ -64,9 +64,7 @@ public class AccountHealingTrackingTest { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG); private WorldStateStorageCoordinator worldStateStorageCoordinator; - private WorldStateProofProvider worldStateProofProvider; - private MerkleTrie accountStateTrie; @Mock SnapWorldDownloadState snapWorldDownloadState; @@ -82,9 +80,7 @@ public void setup() { } @Test - void avoidMarkingAccountWhenStorageProofValid() { - - // generate valid proof + void shouldMarkAccountForHealingWhenStorageProofIsReceived() { final Hash accountHash = Hash.hash(accounts.get(0)); final StateTrieAccountValue stateTrieAccountValue = StateTrieAccountValue.readFrom(RLP.input(accountStateTrie.get(accountHash).orElseThrow())); @@ -108,7 +104,7 @@ void avoidMarkingAccountWhenStorageProofValid() { root -> RangeStorageEntriesCollector.collectEntries( collector, visitor, root, Hash.ZERO)); - // generate the proof + final List proofs = worldStateProofProvider.getStorageProofRelatedNodes( Hash.wrap(storageTrie.getRootHash()), accountHash, Hash.ZERO); @@ -127,11 +123,53 @@ void avoidMarkingAccountWhenStorageProofValid() { snapWorldDownloadState, worldStateProofProvider, slots, new ArrayDeque<>(proofs)); storageRangeDataRequest.getChildRequests( snapWorldDownloadState, worldStateStorageCoordinator, null); + + verify(snapWorldDownloadState).addAccountToHealingList(any(Bytes.class)); + } + + @Test + void shouldNotMarkAccountForHealingWhenAllStorageIsReceivedWithoutProof() { + final Hash accountHash = Hash.hash(accounts.get(0)); + final StateTrieAccountValue stateTrieAccountValue = + StateTrieAccountValue.readFrom(RLP.input(accountStateTrie.get(accountHash).orElseThrow())); + + final StoredMerklePatriciaTrie storageTrie = + new StoredMerklePatriciaTrie<>( + new StoredNodeFactory<>( + (location, hash) -> + worldStateKeyValueStorage.getAccountStorageTrieNode( + accountHash, location, hash), + Function.identity(), + Function.identity()), + stateTrieAccountValue.getStorageRoot()); + + final RangeStorageEntriesCollector collector = + RangeStorageEntriesCollector.createCollector(Hash.ZERO, MAX_RANGE, 10, Integer.MAX_VALUE); + final TrieIterator visitor = RangeStorageEntriesCollector.createVisitor(collector); + final TreeMap slots = + (TreeMap) + storageTrie.entriesFrom( + root -> + RangeStorageEntriesCollector.collectEntries( + collector, visitor, root, Hash.ZERO)); + + final StorageRangeDataRequest storageRangeDataRequest = + SnapDataRequest.createStorageRangeDataRequest( + Hash.wrap(accountStateTrie.getRootHash()), + accountHash, + storageTrie.getRootHash(), + Hash.ZERO, + MAX_RANGE); + storageRangeDataRequest.addResponse( + snapWorldDownloadState, worldStateProofProvider, slots, new ArrayDeque<>()); + storageRangeDataRequest.getChildRequests( + snapWorldDownloadState, worldStateStorageCoordinator, null); + verify(snapWorldDownloadState, never()).addAccountToHealingList(any(Bytes.class)); } @Test - void markAccountOnInvalidStorageProof() { + void shouldMarkAccountForHealingOnInvalidStorageProof() { final Hash accountHash = Hash.hash(accounts.get(0)); final StateTrieAccountValue stateTrieAccountValue = StateTrieAccountValue.readFrom(RLP.input(accountStateTrie.get(accountHash).orElseThrow())); @@ -157,8 +195,7 @@ void markAccountOnInvalidStorageProof() { } @Test - void markAccountOnPartialStorageRange() { - // generate valid proof + void shouldMarkAccountForHealingOnInvalidStorageWithoutProof() { final Hash accountHash = Hash.hash(accounts.get(0)); final StateTrieAccountValue stateTrieAccountValue = StateTrieAccountValue.readFrom(RLP.input(accountStateTrie.get(accountHash).orElseThrow())); @@ -174,11 +211,46 @@ void markAccountOnPartialStorageRange() { stateTrieAccountValue.getStorageRoot()); final RangeStorageEntriesCollector collector = - RangeStorageEntriesCollector.createCollector( + RangeStorageEntriesCollector.createCollector(Hash.ZERO, MAX_RANGE, 1, Integer.MAX_VALUE); + final TrieIterator visitor = RangeStorageEntriesCollector.createVisitor(collector); + final TreeMap slots = + (TreeMap) + storageTrie.entriesFrom( + root -> + RangeStorageEntriesCollector.collectEntries( + collector, visitor, root, Hash.ZERO)); + + final StorageRangeDataRequest storageRangeDataRequest = + SnapDataRequest.createStorageRangeDataRequest( + Hash.wrap(accountStateTrie.getRootHash()), + accountHash, + storageTrie.getRootHash(), Hash.ZERO, - MAX_RANGE, - 1, - Integer.MAX_VALUE); // limit to 1 in order to have a partial range + MAX_RANGE); + storageRangeDataRequest.addResponse( + snapWorldDownloadState, worldStateProofProvider, slots, new ArrayDeque<>()); + + verify(snapWorldDownloadState).addAccountToHealingList(any(Bytes.class)); + } + + @Test + void shouldMarkAccountForHealingOnPartialStorageRange() { + final Hash accountHash = Hash.hash(accounts.get(0)); + final StateTrieAccountValue stateTrieAccountValue = + StateTrieAccountValue.readFrom(RLP.input(accountStateTrie.get(accountHash).orElseThrow())); + + final StoredMerklePatriciaTrie storageTrie = + new StoredMerklePatriciaTrie<>( + new StoredNodeFactory<>( + (location, hash) -> + worldStateKeyValueStorage.getAccountStorageTrieNode( + accountHash, location, hash), + Function.identity(), + Function.identity()), + stateTrieAccountValue.getStorageRoot()); + + final RangeStorageEntriesCollector collector = + RangeStorageEntriesCollector.createCollector(Hash.ZERO, MAX_RANGE, 1, Integer.MAX_VALUE); final TrieIterator visitor = RangeStorageEntriesCollector.createVisitor(collector); final TreeMap slots = (TreeMap) @@ -186,7 +258,7 @@ void markAccountOnPartialStorageRange() { root -> RangeStorageEntriesCollector.collectEntries( collector, visitor, root, Hash.ZERO)); - // generate the proof + final List proofs = worldStateProofProvider.getStorageProofRelatedNodes( Hash.wrap(storageTrie.getRootHash()), accountHash, Hash.ZERO); @@ -205,14 +277,14 @@ void markAccountOnPartialStorageRange() { snapWorldDownloadState, worldStateProofProvider, slots, new ArrayDeque<>(proofs)); verify(snapWorldDownloadState, never()).addAccountToHealingList(any(Bytes.class)); - // should mark during the getchild request storageRangeDataRequest.getChildRequests( snapWorldDownloadState, worldStateStorageCoordinator, null); + verify(snapWorldDownloadState).addAccountToHealingList(any(Bytes.class)); } @Test - void avoidMarkingAccountOnValidStorageTrieNodeDetection() { + void shouldNotMarkAccountForHealingOnValidStorageTrieNodeDetection() { final Hash accountHash = Hash.hash(accounts.get(0)); final StateTrieAccountValue stateTrieAccountValue = StateTrieAccountValue.readFrom(RLP.input(accountStateTrie.get(accountHash).orElseThrow())); @@ -223,6 +295,7 @@ void avoidMarkingAccountOnValidStorageTrieNodeDetection() { Hash.wrap(accountStateTrie.getRootHash()), Bytes.EMPTY); storageTrieNodeHealingRequest.getExistingData(worldStateStorageCoordinator); + verify(snapWorldDownloadState, never()).addAccountToHealingList(any(Bytes.class)); } } From 94099d18f5e4901b8e66c8d7e88662aedbd3cd3b Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Wed, 2 Oct 2024 14:32:48 +0200 Subject: [PATCH 8/8] Add suport for --plugins option in acceptance tests (#7713) Signed-off-by: Fabio Di Fabio --- .../besu/tests/acceptance/dsl/node/BesuNode.java | 7 +++++++ .../dsl/node/ProcessBesuNodeRunner.java | 5 +++++ .../acceptance/dsl/node/ThreadBesuNodeRunner.java | 15 +++++++++++++-- .../node/configuration/BesuNodeConfiguration.java | 7 +++++++ .../BesuNodeConfigurationBuilder.java | 8 ++++++++ .../dsl/node/configuration/BesuNodeFactory.java | 1 + 6 files changed, 41 insertions(+), 2 deletions(-) diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java index 157421531bb..4e48be0401f 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java @@ -128,6 +128,7 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable private boolean useWsForJsonRpc = false; private String token = null; private final List plugins = new ArrayList<>(); + private final List requestedPlugins; private final List extraCLIOptions; private final List staticNodes; private boolean isDnsEnabled = false; @@ -163,6 +164,7 @@ public BesuNode( final boolean secp256k1Native, final boolean altbn128Native, final List plugins, + final List requestedPlugins, final List extraCLIOptions, final List staticNodes, final boolean isDnsEnabled, @@ -224,6 +226,7 @@ public BesuNode( LOG.error("Could not find plugin \"{}\" in resources", pluginName); } }); + this.requestedPlugins = requestedPlugins; engineRpcConfiguration.ifPresent( config -> MergeConfigOptions.setMergeEnabled(config.isEnabled())); this.extraCLIOptions = extraCLIOptions; @@ -738,6 +741,10 @@ public List getPlugins() { return plugins; } + public List getRequestedPlugins() { + return requestedPlugins; + } + @Override public List getExtraCLIOptions() { return extraCLIOptions; diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java index 6e00701ef2b..62951a442db 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java @@ -451,6 +451,11 @@ private List commandlineArgs(final BesuNode node, final Path dataDir) { params.add("--logging=" + level); } + if (!node.getRequestedPlugins().isEmpty()) { + params.add( + "--plugins=" + node.getRequestedPlugins().stream().collect(Collectors.joining(","))); + } + params.addAll(node.getRunCommand()); return params; } diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java index 90de0b0e952..d4334fd93cf 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java @@ -39,6 +39,7 @@ import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.plugins.PluginConfiguration; +import org.hyperledger.besu.ethereum.core.plugins.PluginInfo; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.BlobCacheModule; @@ -302,6 +303,12 @@ public List provideExtraCLIOptions() { return toProvide.getExtraCLIOptions(); } + @Provides + @Named("RequestedPlugins") + public List provideRequestedPlugins() { + return toProvide.getRequestedPlugins(); + } + @Provides Path provideDataDir() { return toProvide.homeDirectory(); @@ -469,7 +476,8 @@ public BesuPluginContextImpl providePluginContext( final RpcEndpointServiceImpl rpcEndpointServiceImpl, final BesuConfiguration commonPluginConfiguration, final PermissioningServiceImpl permissioningService, - final @Named("ExtraCLIOptions") List extraCLIOptions) { + final @Named("ExtraCLIOptions") List extraCLIOptions, + final @Named("RequestedPlugins") List requestedPlugins) { final CommandLine commandLine = new CommandLine(CommandSpec.create()); final BesuPluginContextImpl besuPluginContext = new BesuPluginContextImpl(); besuPluginContext.addService(StorageService.class, storageService); @@ -504,7 +512,10 @@ public BesuPluginContextImpl providePluginContext( besuPluginContext.addService(PrivacyPluginService.class, new PrivacyPluginServiceImpl()); besuPluginContext.initialize( - new PluginConfiguration.Builder().pluginsDir(pluginsPath).build()); + new PluginConfiguration.Builder() + .pluginsDir(pluginsPath) + .requestedPlugins(requestedPlugins.stream().map(PluginInfo::new).toList()) + .build()); besuPluginContext.registerPlugins(); commandLine.parseArgs(extraCLIOptions.toArray(new String[0])); diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java index c77a6c8ac7b..ba69e4ddd72 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java @@ -64,6 +64,7 @@ public class BesuNodeConfiguration { private final boolean secp256k1Native; private final boolean altbn128Native; private final List plugins; + private final List requestedPlugins; private final List extraCLIOptions; private final List staticNodes; private final boolean isDnsEnabled; @@ -102,6 +103,7 @@ public class BesuNodeConfiguration { final boolean secp256k1Native, final boolean altbn128Native, final List plugins, + final List requestedPlugins, final List extraCLIOptions, final List staticNodes, final boolean isDnsEnabled, @@ -137,6 +139,7 @@ public class BesuNodeConfiguration { this.secp256k1Native = secp256k1Native; this.altbn128Native = altbn128Native; this.plugins = plugins; + this.requestedPlugins = requestedPlugins; this.extraCLIOptions = extraCLIOptions; this.staticNodes = staticNodes; this.isDnsEnabled = isDnsEnabled; @@ -239,6 +242,10 @@ public List getPlugins() { return plugins; } + public List getRequestedPlugins() { + return requestedPlugins; + } + public List getExtraCLIOptions() { return extraCLIOptions; } diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java index d8b5e0f9040..ead01ce97d2 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java @@ -93,6 +93,7 @@ public class BesuNodeConfigurationBuilder { private boolean secp256K1Native = true; private boolean altbn128Native = true; private final List plugins = new ArrayList<>(); + private final List requestedPlugins = new ArrayList<>(); private final List extraCLIOptions = new ArrayList<>(); private List staticNodes = new ArrayList<>(); private boolean isDnsEnabled = false; @@ -448,6 +449,12 @@ public BesuNodeConfigurationBuilder plugins(final List plugins) { return this; } + public BesuNodeConfigurationBuilder requestedPlugins(final List requestedPlugins) { + this.requestedPlugins.clear(); + this.requestedPlugins.addAll(requestedPlugins); + return this; + } + public BesuNodeConfigurationBuilder extraCLIOptions(final List extraCLIOptions) { this.extraCLIOptions.clear(); this.extraCLIOptions.addAll(extraCLIOptions); @@ -545,6 +552,7 @@ public BesuNodeConfiguration build() { secp256K1Native, altbn128Native, plugins, + requestedPlugins, extraCLIOptions, staticNodes, isDnsEnabled, diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java index 0451d2a7212..b39274c3d9c 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java @@ -77,6 +77,7 @@ public BesuNode create(final BesuNodeConfiguration config) throws IOException { config.isSecp256k1Native(), config.isAltbn128Native(), config.getPlugins(), + config.getRequestedPlugins(), config.getExtraCLIOptions(), config.getStaticNodes(), config.isDnsEnabled(),