Merge pull request #596 from blxdyx/beta_1

Upstream beta-1

.github/workflows/backups-dashboards.yml

@@ -49,7 +49,7 @@ jobs:
     strategy:
       matrix:
         # For each dashboard add an entry in the list here below (and add an entry in the dictionary within the backup script too!)
-        dashboard: [erigon_custom_metrics]
+        dashboard: [erigon_custom_metrics,erigonQA]
     env:
       DASHBOARDS_AUTH_TOKEN: ${{ secrets.DASHBOARDS_AUTH_TOKEN }}
       GH_TOKEN: ${{ secrets.GH_TOKEN }}

.github/workflows/ci.yml

@@ -12,6 +12,7 @@ on:
       - reopened
       - synchronize
       - ready_for_review
+
   workflow_dispatch:
 
 concurrency:

.github/workflows/kurtosis/pectra.io

@@ -3,10 +3,10 @@ participants_matrix:
     - el_type: erigon
       el_image: test/erigon:current
   cl:
-    - cl_type: lighthouse
-      cl_image: ethpandaops/lighthouse:pawan-electra-alpha7-0dd215c
     - cl_type: teku
-      cl_image: ethpandaops/teku:mekong
+      cl_image: consensys/teku:develop
+    - cl_type: lodestar
+      cl_image: ethpandaops/lodestar:devnet-5-1c2b5ed
 network_params:
   electra_fork_epoch: 1
   min_validator_withdrawability_delay: 1
@@ -19,5 +19,4 @@ assertoor_params:
   run_stability_check: false
   run_block_proposal_check: true
   tests:
-    - { file: "https://raw.githubusercontent.com/ethpandaops/assertoor/refs/heads/electra-support/playbooks/pectra-dev/kurtosis/all.yaml"}
-
+    - { file: "https://raw.githubusercontent.com/ethpandaops/assertoor/refs/heads/electra-support/playbooks/pectra-dev/kurtosis/all.yaml"}

.github/workflows/release.yml

@@ -3,6 +3,10 @@ run-name: Build release ${{ inputs.release_version}} from branch ${{ inputs.chec
 
 env:
   APPLICATION: "bsc-erigon"
+  APPLICATION_VERSION: "Erigon3"
+  TEST_TRACKING_TIME_SECONDS: 7200 # 2 hours
+  TEST_TOTAL_TIME_SECONDS: 432000   # 5 days
+  TEST_CHAIN: "bsc"
   BUILDER_IMAGE: "golang:1.22-bookworm"
   DOCKER_BASE_IMAGE: "debian:12.8-slim"
   APP_REPO: "node-real/bsc-erigon"
@@ -127,7 +131,7 @@ jobs:
           -w /erigon --entrypoint /bin/bash
           ${{ env.BUILDER_IMAGE }}
           -c "git config --global --add safe.directory /erigon;
-          make GOARCH=amd64 GOAMD64=v1 GOBIN=/erigon-build BUILD_TAGS=nosqlite,noboltdb,nosilkworm 
+          make GOARCH=amd64 GOAMD64=v1 GOAMD64_VERSION=v1 GOBIN=/erigon-build BUILD_TAGS=nosqlite,noboltdb,nosilkworm 
           erigon downloader devnet evm caplin diag integration rpcdaemon sentry txpool;
           if [ ${WITH_SILKWORM} ]; then find / -name libsilkworm_capi.so -exec install {} /erigon-build \; ; fi;"
 
@@ -194,6 +198,88 @@ jobs:
           if-no-files-found: error
 
 
+  test-release:
+    name: test on ${{ matrix.id }}
+    runs-on: [ self-hosted, Release, "${{ matrix.runner-arch }}" ]
+    timeout-minutes: 7200  # 5 days
+    needs: [ build-release ]
+    strategy:
+      matrix:
+        include:
+          - id: linux/amd64
+            runner-arch: X64
+            artifact: linux_amd64
+          - id: linux/arm64
+            runner-arch: ARM64
+            artifact: linux_arm64
+
+    steps:
+
+      - name: Cleanup working directory
+        run: rm -drfv *
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Download artifact ${{ env.APPLICATION }}_${{ inputs.release_version }}_${{ matrix.artifact }}.tar.gz
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_${{ matrix.artifact }}.tar.gz
+          path: .
+
+      - name: Extract artifact ${{ env.APPLICATION }}_${{ inputs.release_version }}_${{ matrix.artifact }}.tar.gz
+        run: |
+          pwd
+          ls -l ${{ env.APPLICATION }}_${{ inputs.release_version }}_${{ matrix.artifact }}.tar.gz
+          tar xzvf ${{ env.APPLICATION }}_${{ inputs.release_version }}_${{ matrix.artifact }}.tar.gz
+          ls -lR
+      - name: Fast checkout git repository erigontech/erigon-qa
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 ## 4.1.7 release
+        with:
+          token: ${{ secrets.ORG_GITHUB_ERIGONTECH_ERIGON_QA_READ }}
+          repository: erigontech/erigon-qa
+          fetch-depth: 1
+          ref: main
+          path: erigon-qa
+
+      - name: Checkout QA Tests Repository & Install Requirements
+        run: |
+          cd ./erigon-qa/test_system
+          pwd
+          ls -lao
+          pip3 install -r requirements.txt   
+          ln -s $(pwd)/base_library $(pwd)/qa-tests/tip-tracking/base_library
+          echo "DEBUG -- content of directory $(pwd) :"
+          ls -l
+          echo "DEBUG -- content of directory $(pwd)/qa-tests/tip-tracking/"
+          ls -l $(pwd)/qa-tests/tip-tracking/
+          echo "DEBUG -- content of directory GITHUB_WORKSPACE ${GITHUB_WORKSPACE} :"
+          ls -l ${GITHUB_WORKSPACE}
+          echo "DEBUG -- end."
+          rm -rf ${RUNNER_WORKSPACE}/erigon-data || true
+          mkdir ${RUNNER_WORKSPACE}/erigon-data
+          # Run Erigon, wait sync and check ability to maintain sync
+          python3 qa-tests/tip-tracking/run_and_check_tip_tracking.py \
+            ${GITHUB_WORKSPACE}/${{ env.APPLICATION }}_${{ inputs.release_version }}_${{ matrix.artifact }} \
+              ${RUNNER_WORKSPACE}/erigon-data ${{ env.TEST_TRACKING_TIME_SECONDS }} ${{ env.TEST_TOTAL_TIME_SECONDS }} ${{ env.APPLICATION_VERSION }} ${{ env.TEST_CHAIN }}
+          # Capture monitoring script exit status
+          test_exit_status=$?
+          # Save the subsection reached status
+          echo "::set-output name=test_executed::true"
+          # Check test runner script exit status
+          if [ $test_exit_status -eq 0 ]; then
+            echo "Tests completed successfully"
+            echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
+          else
+            echo "Error detected during tests"
+            echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Cleanup working directory
+        run: rm -drfv *
+
   build-debian-pkg:
     name: Debian packages
     needs: [ build-release ]

.github/workflows/test-hive-eest.yml

@@ -14,8 +14,8 @@ jobs:
       - name: Checkout Hive
         uses: actions/checkout@v4
         with:
-          repository: danceratopz/hive
-          ref: prague-devnet-4
+          repository: ethereum/hive
+          # ref: prague-devnet-4
           path: hive
       - name: Setup go env and cache
         uses: actions/setup-go@v5
@@ -47,7 +47,7 @@ jobs:
             echo -e "\n\n============================================================"
             echo "Running test: ${1}"
             echo -e "\n"
-            ./hive --sim 'ethereum/eest/consume-engine' --client erigon 2>&1 | tee output.log || {
+            ./hive --sim 'ethereum/eest/consume-engine' --client erigon --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/pectra-devnet-5%40v1.1.0/fixtures_pectra-devnet-5.tar.gz --sim.buildarg branch=pectra-devnet-5 2>&1 | tee output.log || {
               if [ $? -gt 0 ]; then
                 echo "Exitcode gt 0"
               fi

.github/workflows/test-integration-caplin.yml

@@ -13,7 +13,7 @@ on:
       - reopened
       - synchronize
       - ready_for_review
-
+    
 jobs:
   tests:
     strategy:

.github/workflows/test-integration-erigon.yml

@@ -1,4 +1,5 @@
 name: Integration tests - ERIGON
+
 on:
   push:
     branches:
@@ -17,59 +18,114 @@ on:
   workflow_dispatch:
 
 jobs:
+  #
+  # This first job is used to determine if changes are within out-of-scope dirs or files (in such a case integration tests are not run because they would be meaningless)
+  # NOTE: this logic is needed because the simple 'paths-ignore:' doesn't work since this workflow is set as a mandatory/required check for this repo
+  # - '**/.github/workflows/**' is currently commented to avoid unintended freeze in case of concurrent changes outside the excluded paths (further development will be done in due course)
+  #
+  source-of-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      changed_files: ${{ steps.filter.outputs.changed_files }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Check for changes within out-of-scope dirs or files
+        id: filter
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            changed_files:
+              - 'dashboards/**'
+              # - '**/.github/workflows/**'
+              - '**/.github/workflows/backups-dashboards.yml'
+
   tests-mac-linux:
+    needs: source-of-changes
     strategy:
       matrix:
-        # list of os: https://github.com/actions/virtual-environments
         os:
           - ubuntu-22.04
           - macos-14
-        # - ubuntu-latest-erigontests-large
+          # - ubuntu-latest-erigontests-large
     runs-on: ${{ matrix.os }}
 
     steps:
-      - name: declaring runners
+      - name: Declare runners
+        if: needs.source-of-changes.outputs.changed_files != 'true'
         run: |
           set +x
           echo "I am being served by this runner: $RUNNER_NAME"
-      - uses: actions/checkout@v4
-      - run: git submodule update --init --recursive --force
-      - uses: actions/setup-go@v5
+
+      - name: Checkout code
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        uses: actions/checkout@v4
+
+      - name: Update submodules
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        run: git submodule update --init --recursive --force
+
+      - name: Setup Go environment
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        uses: actions/setup-go@v5
         with:
           go-version: '1.23'
           cache: ${{ contains(fromJSON('["refs/heads/main","refs/heads/release/2.60","refs/heads/release/2.61"]'), github.ref) }}
+
       - name: Install dependencies on Linux
-        if: runner.os == 'Linux'
+        if: runner.os == 'Linux' && needs.source-of-changes.outputs.changed_files != 'true'
         run: sudo apt update && sudo apt install build-essential
-      - name: test-integration
+
+      - name: Run integration tests on ${{ matrix.os }}
+        if: needs.source-of-changes.outputs.changed_files != 'true'
         run: GOGC=80 make test-integration
 
+      - name: This ${{ matrix.os }} check does not make sense for changes within out-of-scope directories
+        if: needs.source-of-changes.outputs.changed_files == 'true'
+        run: echo "This check does not make sense for changes within out-of-scope directories"
+
+
   tests-windows:
+    needs: source-of-changes
     strategy:
       matrix:
         os: [ windows-2022 ]
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v4
-      - run: git submodule update --init --recursive --force
-      - uses: actions/setup-go@v5
+      - name: Checkout code on ${{ matrix.os }}
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        uses: actions/checkout@v4
+
+      - name: Update submodules on Windows
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        run: git submodule update --init --recursive --force
+
+      - name: Setup Go environment on ${{ matrix.os }}
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        uses: actions/setup-go@v5
         with:
           go-version: '1.23'
 
-      - uses: actions/cache@v4
+      - name: Cache Chocolatey packages
+        if: needs.source-of-changes.outputs.changed_files != 'true'
+        uses: actions/cache@v4
         with:
           path: |
             C:\ProgramData\chocolatey\lib\mingw
             C:\ProgramData\chocolatey\lib\cmake
           key: chocolatey-${{ matrix.os }}
-      - name: Install dependencies
+
+      - name: Install dependencies on ${{ matrix.os }}
+        if: needs.source-of-changes.outputs.changed_files != 'true'
         run: |
           choco upgrade mingw -y --no-progress --version 13.2.0
 
-      - name: test-integration
+      - name: Run integration tests on ${{ matrix.os }}
+        if: needs.source-of-changes.outputs.changed_files != 'true'
         run: .\wmake.ps1 test-integration
 
-
-
-
+      - name: This ${{ matrix.os }} check does not make sense for changes within out-of-scope directories
+        if: needs.source-of-changes.outputs.changed_files == 'true'
+        run: echo "This check does not make sense for changes within out-of-scope directories"

.github/workflows/test-kurtosis-assertoor.yml

@@ -39,10 +39,10 @@ jobs:
             kurtosis_extra_args: --verbosity detailed --cli-log-level trace
             persistent_logs: "true"
 
-      - name: Run Pectra Kurtosis + assertoor tests
-        uses: ethpandaops/kurtosis-assertoor-github-action@v1
-        with:
-            enclave_name: "kurtosis-run2-${{ github.run_id }}"
-            ethereum_package_args: ".github/workflows/kurtosis/pectra.io"
-            kurtosis_extra_args: --verbosity detailed --cli-log-level trace
-            persistent_logs: "true"
+      # - name: Run Pectra Kurtosis + assertoor tests
+      #   uses: ethpandaops/kurtosis-assertoor-github-action@v1
+      #   with:
+      #       enclave_name: "kurtosis-run2-${{ github.run_id }}"
+      #       ethereum_package_args: ".github/workflows/kurtosis/pectra.io"
+      #       kurtosis_extra_args: --verbosity detailed --cli-log-level trace
+      #       persistent_logs: "true"

ChangeLog.md

@@ -5,17 +5,11 @@ ChangeLog
 
 ### Breaking changes
 
-- `--prune.mode` default is `full`. For compatibility with `geth` and other clients. Plz set explicit
-  `--prune.mode` flag to your Erigon3 setups - to simplify future upgrade/downgrade.
 - Bor chains: enable our internal Consensus Layer by default (name: Astrid)
-- Released binaries for Linux are now dynamically linked (
-  Issues: https://github.com/erigontech/erigon/issues/12570, https://github.com/erigontech/erigon/issues/12556 )
-
-### New features:
-
-- Decided to fix snapshots format and go for `beta1`. Main focus: fast bug-reports fixes, chain-tip/rpc perf, validator
-  mode.
-- Erigon3 book: https://development.erigon-documentation-preview.pages.dev/
+    - The process should auto upgrade - in which case you may find that it starts creating new snapshots for checkpoints and milestones.
+    - This may however fail, as there are a number of potential edge cases.  If this happens the process will likely stop with a failure message.
+    - In this situation you will need to do a clean sync, in which case the complete snapshot set will be downloaded and astrid will sync.
+    - If you want to prevent this and retain the old behaviour start erigon with --polygon.sync=false
 
 ### TODO
 

Dockerfile.release

@@ -95,7 +95,7 @@ ARG USER=erigon \
     EXPOSED_PORTS
 
 RUN --mount=type=bind,from=ci-cd-main-branch-builder,source=/build-${TARGETARCH},target=/tmp/erigon \
-    apk add --no-cache ca-certificates tzdata libstdc++ && \
+    apk add --no-cache curl ca-certificates tzdata libstdc++ && \
     addgroup ${GROUP} && \
     adduser -D -h /home/${USER} -G ${GROUP} ${USER} && \
     install -d -o ${USER} -g ${GROUP} /home/${USER}/.local /home/${USER}/.local/share /home/${USER}/.local/share/erigon && \

README.md

@@ -1,8 +1,8 @@
 # Erigon
 
 Documentation: **[docs.erigon.tech](https://docs.erigon.tech)**
-Blog: **[erigon.substack.com](https://erigon.substack.com/)**
-Twitter: [x.com/ErigonEth](https://x.com/ErigonEth)
+Blog: **[erigon.tech/news](https://erigon.tech/news/)**
+X/Twitter: **[x.com/ErigonEth](https://x.com/ErigonEth)**
 
 Erigon is an implementation of Ethereum (execution layer with embeddable consensus layer), on the efficiency
 frontier.
@@ -722,14 +722,6 @@ Getting in touch
 The main discussions are happening on our Discord server. To get an invite, send an email to `bloxster [at] proton.me`
 with your name, occupation, a brief explanation of why you want to join the Discord, and how you heard about Erigon.
 
-### Blog
-
-**[erigon.substack.com](https://erigon.substack.com/)**
-
-### Twitter
-
-[x.com/ErigonEth](https://x.com/ErigonEth)
-
 ### Reporting security issues/concerns
 
 Send an email to `security [at] torquem.ch`.

accounts/abi/bind/backends/simulated.go

@@ -176,7 +176,7 @@ func (b *SimulatedBackend) emptyPendingBlock() {
 	b.pendingBlock = blockChain.Blocks[0]
 	b.pendingReceipts = blockChain.Receipts[0]
 	b.pendingHeader = blockChain.Headers[0]
-	b.gasPool = new(core.GasPool).AddGas(b.pendingHeader.GasLimit).AddBlobGas(b.m.ChainConfig.GetMaxBlobGasPerBlock())
+	b.gasPool = new(core.GasPool).AddGas(b.pendingHeader.GasLimit).AddBlobGas(b.m.ChainConfig.GetMaxBlobGasPerBlock(b.pendingHeader.Time))
 	if b.pendingReaderTx != nil {
 		b.pendingReaderTx.Rollback()
 	}