CI: revert PRs to see impact on test time

.github/workflows/ci-pr.yml

@@ -24,14 +24,9 @@ permissions:
 
 jobs:
   test:
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: ["ubuntu-24.04"]
-        partition_id: [0, 1, 2, 3, 4, 5] # set PARTITION_TOTAL below to match
-    runs-on: ${{ matrix.platform }}
+    runs-on: ubuntu-24.04
     env:
-      PARTITION_ID: ${{ matrix.partition_id }}
+      PARTITION_ID: 3
       PARTITION_TOTAL: 6
       CIRCLECI: true
       SHORTTEST: "-short"
@@ -48,271 +43,23 @@ jobs:
         uses: ./.github/actions/setup-test
       - name: Run tests
         run: |
-          PACKAGES="$(go list ./... | grep -v /go-algorand/test/)"
-          export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')          
-          mkdir -p test_results/${{ matrix.platform }}_test/${PARTITION_ID}
+          mkdir -p /home/runner/test_results/ubuntu-24.04_test/${PARTITION_ID}
           gotestsum --format standard-quiet \
-            --junitfile ~/test_results/${{ matrix.platform }}_test/${PARTITION_ID}/results.xml \
-            --jsonfile ~/test_results/${{ matrix.platform }}_test/${PARTITION_ID}/testresults.json \
+            --junitfile /home/runner/test_results/ubuntu-24.04_test/${PARTITION_ID}/results.xml \
+            --jsonfile /home/runner/test_results/ubuntu-24.04_test/${PARTITION_ID}/testresults.json \
             -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" $SHORTTEST \
             -race -timeout 1h -coverprofile=coverage.txt -covermode=atomic -p 4 \
-            $PACKAGE_NAMES
-      - name: Notify Slack on failure
-        if: failure()
-        uses: ./.github/actions/slack-notify
-        with:
-          job-type: "Test"
-          build-type: "PR Build"
-          details: "• Partition: `${{ matrix.partition_id }}` of ${{ env.PARTITION_TOTAL }}\n• Failed Step: `${{ steps.run_tests.name }}`"
-      - name: Upload test logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v4
-        with:
-          name: test-logs-${{ matrix.platform }}-${{ github.run_id }}-${{ matrix.partition_id }}
-          path: |
-            **/*.log
-          retention-days: 30
-      - name: Upload test artifacts to GitHub
-        if: ${{ !cancelled() }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: test-results-${{ matrix.platform }}-${{ github.run_id }}-${{ matrix.partition_id }}
-          path: ~/test_results
-          retention-days: 7
-      - name: Upload coverage
-        # Only upload coverage from ubuntu-24.04 platform
-        if: matrix.platform == 'ubuntu-24.04' && ${{ !cancelled() }}
-        uses: codecov/codecov-action@v4
-        env:
-          GITHUB_ACTIONS: True
-          CIRCLECI: ""
-        with:
-          token: ${{ env.CODECOV_TOKEN }}
-          file: ./coverage.txt
-          fail_ci_if_error: false
-      - name: Upload test results to Codecov
-        if: ${{ !cancelled() }}
-        uses: codecov/test-results-action@v1
-        with:
-          file: ${{ matrix.platform == 'macos-14' && '/Users/runner' || '/home/runner' }}/test_results/${{ matrix.platform }}_test/${{ matrix.partition_id }}/results.xml
-          token: ${{ env.CODECOV_TOKEN }}
-          fail_ci_if_error: false
-
-  integration:
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: ["ubuntu-24.04"]
-        partition_id: [0, 1, 2, 3, 4] # set PARTITION_TOTAL below to match
-    runs-on: ${{ matrix.platform }}
-    env:
-      CIRCLECI: true
-      PARTITION_ID: ${{ matrix.partition_id }}
-      PARTITION_TOTAL: 5
-      E2E_TEST_FILTER: GO
-      PARALLEL_FLAG: "-p 4"
-      SHORTTEST: "-short"
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Set up Go
-        uses: ./.github/actions/setup-go
-        with:
-          cache-prefix: build-e2e
-      - name: Setup test environment
-        uses: ./.github/actions/setup-test
-      - name: Build e2e binaries
-        run: make build-e2e
-      - name: Run integration tests
-        run: |
-          mkdir -p ~/test_results/${{ matrix.platform }}_integration/${PARTITION_ID}
-          NO_BUILD=true TEST_RESULTS=~/test_results/${{ matrix.platform }}_integration/${PARTITION_ID} \
-          test/scripts/e2e.sh
-      - name: Notify Slack on failure
-        if: failure()
-        uses: ./.github/actions/slack-notify
-        with:
-          job-type: "Integration Test"
-          build-type: "PR Build"
-          details: "• Partition: `${{ matrix.partition_id }}` of ${{ env.PARTITION_TOTAL }}\n• Failed Step: `${{ steps.run_integration_tests.name }}`"
-      - name: Upload test artifacts to GitHub
-        if: ${{ !cancelled() }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: integration-results-${{ matrix.platform }}-${{ github.run_id }}-${{ matrix.partition_id }}
-          path: ~/test_results
-          retention-days: 7
-      - name: Upload test results to Codecov
-        if: ${{ !cancelled() }}
-        uses: codecov/test-results-action@v1
-        with:
-          file: ${{ matrix.platform == 'macos-14' && '/Users/runner' || '/home/runner' }}/test_results/${{ matrix.platform }}_integration/${{ matrix.partition_id }}/results.xml
-          token: ${{ env.CODECOV_TOKEN }}
-          fail_ci_if_error: false
-
-  e2e_expect:
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: ["ubuntu-24.04"]
-        partition_id: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # set PARTITION_TOTAL below to match
-    runs-on: ${{ matrix.platform }}
-    env:
-      CIRCLECI: true
-      PARTITION_ID: ${{ matrix.partition_id }}
-      PARTITION_TOTAL: 10
-      E2E_TEST_FILTER: EXPECT
-      PARALLEL_FLAG: "-p 4"
-      SHORTTEST: "-short"
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Set up Go
-        uses: ./.github/actions/setup-go
-        with:
-          cache-prefix: build-e2e
-      - name: Setup test environment
-        uses: ./.github/actions/setup-test
-        with:
-          install-expect: true
-      - name: Build e2e binaries
-        run: make build-e2e
-      - name: Run E2E expect tests
-        run: |
-          mkdir -p ~/test_results/${{ matrix.platform }}_e2e_expect/${PARTITION_ID}
-          NO_BUILD=true TEST_RESULTS=~/test_results/${{ matrix.platform }}_e2e_expect/${PARTITION_ID} \
-          test/scripts/e2e.sh
-      - name: Notify Slack on failure
-        if: failure()
-        uses: ./.github/actions/slack-notify
-        with:
-          job-type: "Expect Test"
-          build-type: "PR Build"
-          details: "• Partition: `${{ matrix.partition_id }}` of ${{ env.PARTITION_TOTAL }}\n• Failed Step: `${{ steps.run_e2e_expect_tests.name }}`"
-      - name: Upload test artifacts to GitHub
-        if: ${{ !cancelled() }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: e2e_expect-results-${{ matrix.platform }}-${{ github.run_id }}-${{ matrix.partition_id }}
-          path: ~/test_results
-          retention-days: 7
-      - name: Upload test results to Codecov
-        if: ${{ !cancelled() }}
-        uses: codecov/test-results-action@v1
-        with:
-          file: ${{ matrix.platform == 'macos-14' && '/Users/runner' || '/home/runner' }}/test_results/${{ matrix.platform }}_e2e_expect/${{ matrix.partition_id }}/results.xml
-          token: ${{ env.CODECOV_TOKEN }}
-          fail_ci_if_error: false
-
-  e2e_subs:
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: ["ubuntu-24.04"]
-        testsuite: ["parallel", "vdir-serial"]
-    runs-on: ${{ matrix.platform }}
-    env:
-      E2E_TEST_FILTER: SCRIPTS
-      E2E_SUBS_TESTSUITE: ${{ matrix.testsuite }}
-      CI_PLATFORM: ${{ matrix.platform }}
-      CI_KEEP_TEMP_PLATFORM: ""
-      SHORTTEST: "-short"
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Set up Go
-        uses: ./.github/actions/setup-go
-        with:
-          cache-prefix: buildsrc
-      - name: Setup test environment
-        uses: ./.github/actions/setup-test
-      - name: Build binaries
-        run: make buildsrc
-      - name: Run E2E subs tests
-        run: |
-          mkdir -p ~/test_results/${{ matrix.platform }}_e2e_subs
-          NO_BUILD=true TEST_RESULTS=~/test_results/${{ matrix.platform }}_e2e_subs \
-          test/scripts/e2e.sh
-      - name: Notify Slack on failure
-        if: failure()
-        uses: ./.github/actions/slack-notify
-        with:
-          job-type: "Subs Test"
-          build-type: "PR Build"
-          details: "• Test Suite: `${{ matrix.testsuite }}`\n• Failed Step: `${{ steps.run_e2e_expect_tests.name }}`"
+            -cpuprofile=/home/runner/test_results/ubuntu-24.04_test/${PARTITION_ID}/cpu.prof \
+            -memprofile=/home/runner/test_results/ubuntu-24.04_test/${PARTITION_ID}/mem.prof \
+            ./agreement/
+          # Generate text reports from profiles
+          go tool pprof -top -cum /home/runner/test_results/ubuntu-24.04_test/${PARTITION_ID}/cpu.prof > /home/runner/test_results/ubuntu-24.04_test/${PARTITION_ID}/cpu_profile.txt || true
+          go tool pprof -top -sample_index=alloc_space /home/runner/test_results/ubuntu-24.04_test/${PARTITION_ID}/mem.prof > /home/runner/test_results/ubuntu-24.04_test/${PARTITION_ID}/mem_profile.txt || true
       - name: Upload test artifacts to GitHub
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: e2e_subs-results-${{ matrix.platform }}-${{ github.run_id }}-${{ matrix.testsuite }}
-          path: ~/test_results
+          name: test-results-partition-3-${{ github.run_id }}
+          path: /home/runner/test_results
           retention-days: 7
 
-  verify:
-    needs: [test, integration, e2e_expect, e2e_subs]
-    strategy:
-      fail-fast: false
-      matrix:
-        test_type: ["test", "integration", "e2e_expect", "e2e_subs"]
-        platform: ["ubuntu-24.04"]
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: ${{ matrix.test_type }}-results-${{ matrix.platform }}-*
-          path: ~/test_results
-          merge-multiple: true
-      - name: Check test execution
-        run: |
-          cat ~/test_results/${{ matrix.platform }}_${{ matrix.test_type }}/**/testresults.json > ~/test_results/${{ matrix.platform }}_${{ matrix.test_type }}/combined_testresults.json
-          python3 scripts/buildtools/check_tests.py \
-            --tests-results-filepath ~/test_results/${{ matrix.platform }}_${{ matrix.test_type }}/combined_testresults.json \
-            --ignored-tests \
-              TestAlgodWithExpect \
-              TestAlgohWithExpect \
-              TestGoalWithExpect \
-              TestTealdbgWithExpect
-      - name: Notify Slack on failure
-        if: failure()
-        uses: ./.github/actions/slack-notify
-        with:
-          job-type: "Verify"
-          build-type: "PR Build"
-          details: "• Test Type: `${{ matrix.test_type }}`\n• Branch: `${{ github.ref_name }}`"
-
-  report:
-    needs: [test, integration, e2e_expect, e2e_subs]
-    if: always()
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: "*-results-*"
-          path: ~/test_results
-          merge-multiple: true
-      - name: Generate failure summary
-        run: |
-          echo "## Test Results" >> $GITHUB_STEP_SUMMARY
-          for job_type in test integration e2e_expect e2e_subs; do
-            FAILED_TESTS=$(find ~/test_results -path "*_${job_type}/*" -name "testresults.json" -exec jq -r 'select(.Action == "fail" and .Test) | "\(.Package)/\(.Test)"' {} \; 2>/dev/null | sort -u | grep -v '^$' || true)
-            if [ -n "$FAILED_TESTS" ]; then
-              echo "### ${job_type} failures" >> $GITHUB_STEP_SUMMARY
-              echo '```' >> $GITHUB_STEP_SUMMARY
-              echo "$FAILED_TESTS" >> $GITHUB_STEP_SUMMARY
-              echo '```' >> $GITHUB_STEP_SUMMARY
-              echo "FAILED=true" >> $GITHUB_ENV
-            else
-              echo "### ${job_type} ✅" >> $GITHUB_STEP_SUMMARY
-            fi
-          done
-
-      - name: Fail if there were test failures
-        if: env.FAILED == 'true'
-        run: exit 1

agreement/message.go

@@ -18,13 +18,17 @@ package agreement
 
 import (
 	"github.com/algorand/go-algorand/protocol"
+	"github.com/algorand/msgp/msgp"
 )
 
 // A message represents an internal message which is passed between components
 // of the agreement service.
 type message struct {
 	_struct struct{} `codec:","`
 
+	// this field is for backwards compatibility with crash state serialized using go-codec prior to explicit unexport.
+	// should be removed after the next consensus update.
+	MessageHandle msgp.Raw `codec:"MessageHandle,omitempty"`
 	// explicitly unexport this field since we can't define serializers for interface{} type
 	// the only implementation of this is gossip.messageMetadata which doesn't have exported fields to serialize.
 	messageHandle MessageHandle

agreement/message_test.go

@@ -17,14 +17,17 @@
 package agreement
 
 import (
+	"encoding/base64"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/committee"
+	"github.com/algorand/go-algorand/network"
 	"github.com/algorand/go-algorand/protocol"
+	"github.com/algorand/go-algorand/test/partitiontest"
 )
 
 var poolAddr = basics.Address{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
@@ -80,3 +83,52 @@ func BenchmarkVoteDecoding(b *testing.B) {
 		decodeVote(msgBytes)
 	}
 }
+
+// TestMessageBackwardCompatibility ensures MessageHandle field can be
+// properly decoded from message.
+// This test is only needed for agreement state serialization switch from reflection to msgp.
+func TestMessageBackwardCompatibility(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	type messageMetadata struct {
+		raw network.IncomingMessage
+	}
+
+	encoded, err := base64.StdEncoding.DecodeString("iaZCdW5kbGWAr0NvbXBvdW5kTWVzc2FnZYKoUHJvcG9zYWyApFZvdGWArU1lc3NhZ2VIYW5kbGWAqFByb3Bvc2FsgKNUYWeiUFC1VW5hdXRoZW50aWNhdGVkQnVuZGxlgLdVbmF1dGhlbnRpY2F0ZWRQcm9wb3NhbICzVW5hdXRoZW50aWNhdGVkVm90ZYCkVm90ZYA=")
+	require.NoError(t, err)
+
+	// run on master f57a276 to get the encoded data for above
+	// msg := message{
+	// 	MessageHandle: &messageMetadata{raw: network.IncomingMessage{Tag: protocol.Tag("mytag"), Data: []byte("some data")}},
+	// 	Tag:           protocol.ProposalPayloadTag,
+	// }
+
+	// result := protocol.EncodeReflect(&msg)
+	// fmt.Println(base64.StdEncoding.EncodeToString(result))
+
+	// messages for all rounds after this change should not have MessageHandle set so clearing it out and re-encoding/decoding it should yield this
+	targetMessage := message{
+		Tag: protocol.ProposalPayloadTag,
+	}
+
+	require.Containsf(t, string(encoded), "MessageHandle", "encoded message does not contain MessageHandle field")
+	var m1, m2, m3, m4 message
+	// Both msgp and reflection should decode the message containing old MessageHandle successfully
+	err = protocol.Decode(encoded, &m1)
+	require.NoError(t, err)
+	err = protocol.DecodeReflect(encoded, &m2)
+	require.NoError(t, err)
+	// after setting MessageHandle to nil both should re-encode and decode to same values
+	m1.MessageHandle = nil
+	m2.MessageHandle = nil
+	e1 := protocol.Encode(&m1)
+	e2 := protocol.EncodeReflect(&m2)
+	require.Equal(t, e1, e2)
+	require.NotContainsf(t, string(e1), "MessageHandle", "encoded message still contains MessageHandle field")
+	err = protocol.DecodeReflect(e1, &m3)
+	require.NoError(t, err)
+	err = protocol.Decode(e2, &m4)
+	require.NoError(t, err)
+	require.Equal(t, m3, m4)
+	require.Equal(t, m3, targetMessage)
+}