Merge #4417

4417: [Exec] Use CCF in self-describing mode to encode events (replaces JSON-CDC) r=fxamacker a=fxamacker

Updates onflow/cadence#2283 

This PR uses CCF in fully self-describing mode, so events will encode to about 1/2 the size of JSON-CDC encoding.

Using CCF in partially self-describing mode can encode events to 1/14 the size of JSON-CDC but that requires other changes outside of CCF codec and other resulting work.

### TODO

- [x] Update this PR after #4390 is merged.
- [x] After the update, run CI tests and resolve any issues.

:heavy_check_mark: Requires PR #4390 (merged on June 2, 2023).
:heavy_check_mark: Requires PR onflow/cadence#2528
✔️ Requires PR onflow/cadence#2529
✔️ Requires PR onflow/flow-emulator#408

Co-authored-by: Faye Amacker <33205765+fxamacker@users.noreply.github.com>
Co-authored-by: Amlandeep Bhadra <amlandeep1912@gmail.com>

engine/execution/computation/computer/computer_test.go

@@ -8,7 +8,7 @@ import (
 	"testing"
 
 	"github.com/onflow/cadence"
-	"github.com/onflow/cadence/encoding/json"
+	"github.com/onflow/cadence/encoding/ccf"
 	"github.com/onflow/cadence/runtime"
 	"github.com/onflow/cadence/runtime/common"
 	"github.com/onflow/cadence/runtime/interpreter"
@@ -540,7 +540,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) {
 			serviceEvents, err := systemcontracts.ServiceEventsForChain(execCtx.Chain.ChainID())
 			require.NoError(t, err)
 
-			payload, err := json.Decode(nil, []byte(unittest.EpochSetupFixtureJSON))
+			payload, err := ccf.Decode(nil, unittest.EpochSetupFixtureCCF)
 			require.NoError(t, err)
 
 			serviceEventA, ok := payload.(cadence.Event)
@@ -551,7 +551,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) {
 			}
 			serviceEventA.EventType.QualifiedIdentifier = serviceEvents.EpochSetup.QualifiedIdentifier()
 
-			payload, err = json.Decode(nil, []byte(unittest.EpochCommitFixtureJSON))
+			payload, err = ccf.Decode(nil, unittest.EpochCommitFixtureCCF)
 			require.NoError(t, err)
 
 			serviceEventB, ok := payload.(cadence.Event)
@@ -562,7 +562,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) {
 			}
 			serviceEventB.EventType.QualifiedIdentifier = serviceEvents.EpochCommit.QualifiedIdentifier()
 
-			payload, err = json.Decode(nil, []byte(unittest.VersionBeaconFixtureJSON))
+			payload, err = ccf.Decode(nil, unittest.VersionBeaconFixtureCCF)
 			require.NoError(t, err)
 
 			serviceEventC, ok := payload.(cadence.Event)
@@ -1177,7 +1177,7 @@ func Test_ExecutingSystemCollection(t *testing.T) {
 	noopCollector := metrics.NewNoopCollector()
 
 	expectedNumberOfEvents := 3
-	expectedEventSize := 1721
+	expectedEventSize := 1435
 	// bootstrapping does not cache programs
 	expectedCachedPrograms := 0
 

engine/execution/computation/programs_test.go

@@ -10,7 +10,7 @@ import (
 	dssync "github.com/ipfs/go-datastore/sync"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	"github.com/onflow/cadence"
-	jsoncdc "github.com/onflow/cadence/encoding/json"
+	"github.com/onflow/cadence/encoding/ccf"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -543,7 +543,7 @@ func prepareTx(t *testing.T,
 }
 
 func hasValidEventValue(t *testing.T, event flow.Event, value int) {
-	data, err := jsoncdc.Decode(nil, event.Payload)
+	data, err := ccf.Decode(nil, event.Payload)
 	require.NoError(t, err)
 	assert.Equal(t, int16(value), data.(cadence.Event).Fields[0].ToGoValue())
 }

engine/execution/rpc/engine.go

@@ -256,7 +256,8 @@ func (h *handler) GetEventsForBlockIDs(_ context.Context,
 	}
 
 	return &execution.GetEventsForBlockIDsResponse{
-		Results: results,
+		Results:              results,
+		EventEncodingVersion: execution.EventEncodingVersion_CCF_V0,
 	}, nil
 }
 
@@ -316,9 +317,10 @@ func (h *handler) GetTransactionResult(
 
 	// compose a response with the events and the transaction error
 	return &execution.GetTransactionResultResponse{
-		StatusCode:   statusCode,
-		ErrorMessage: errMsg,
-		Events:       events,
+		StatusCode:           statusCode,
+		ErrorMessage:         errMsg,
+		Events:               events,
+		EventEncodingVersion: execution.EventEncodingVersion_CCF_V0,
 	}, nil
 }
 
@@ -374,9 +376,10 @@ func (h *handler) GetTransactionResultByIndex(
 
 	// compose a response with the events and the transaction error
 	return &execution.GetTransactionResultResponse{
-		StatusCode:   statusCode,
-		ErrorMessage: errMsg,
-		Events:       events,
+		StatusCode:           statusCode,
+		ErrorMessage:         errMsg,
+		Events:               events,
+		EventEncodingVersion: execution.EventEncodingVersion_CCF_V0,
 	}, nil
 }
 
@@ -452,7 +455,8 @@ func (h *handler) GetTransactionResultsByBlockID(
 
 	// compose a response
 	return &execution.GetTransactionResultsResponse{
-		TransactionResults: responseTxResults,
+		TransactionResults:   responseTxResults,
+		EventEncodingVersion: execution.EventEncodingVersion_CCF_V0,
 	}, nil
 }
 

engine/execution/testutil/fixtures.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/onflow/cadence"
+	"github.com/onflow/cadence/encoding/ccf"
 	jsoncdc "github.com/onflow/cadence/encoding/json"
 	"github.com/stretchr/testify/require"
 
@@ -281,7 +282,7 @@ func CreateAccountsWithSimpleAddresses(
 
 		for _, event := range output.Events {
 			if event.Type == flow.EventAccountCreated {
-				data, err := jsoncdc.Decode(nil, event.Payload)
+				data, err := ccf.Decode(nil, event.Payload)
 				if err != nil {
 					return snapshotTree, nil, errors.New(
 						"error decoding events")

fvm/accounts_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/onflow/cadence"
+	"github.com/onflow/cadence/encoding/ccf"
 	jsoncdc "github.com/onflow/cadence/encoding/json"
 	"github.com/onflow/cadence/runtime/format"
 	"github.com/stretchr/testify/assert"
@@ -70,7 +71,7 @@ func createAccount(
 
 	require.Len(t, accountCreatedEvents, 1)
 
-	data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload)
+	data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload)
 	require.NoError(t, err)
 	address := flow.ConvertAddress(
 		data.(cadence.Event).Fields[0].(cadence.Address))
@@ -407,7 +408,7 @@ func TestCreateAccount(t *testing.T) {
 				accountCreatedEvents := filterAccountCreatedEvents(output.Events)
 				require.Len(t, accountCreatedEvents, 1)
 
-				data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload)
+				data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload)
 				require.NoError(t, err)
 				address := flow.ConvertAddress(
 					data.(cadence.Event).Fields[0].(cadence.Address))
@@ -450,7 +451,7 @@ func TestCreateAccount(t *testing.T) {
 					}
 					accountCreatedEventCount += 1
 
-					data, err := jsoncdc.Decode(nil, event.Payload)
+					data, err := ccf.Decode(nil, event.Payload)
 					require.NoError(t, err)
 					address := flow.ConvertAddress(
 						data.(cadence.Event).Fields[0].(cadence.Address))

fvm/environment/event_emitter_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/onflow/cadence"
-	jsoncdc "github.com/onflow/cadence/encoding/json"
+	"github.com/onflow/cadence/encoding/ccf"
 	"github.com/onflow/cadence/runtime/common"
 	"github.com/onflow/cadence/runtime/stdlib"
 
@@ -179,7 +179,7 @@ func createTestEventEmitterWithLimit(chain flow.ChainID, address flow.Address, e
 }
 
 func getCadenceEventPayloadByteSize(event cadence.Event) uint64 {
-	payload, err := jsoncdc.Encode(event)
+	payload, err := ccf.Encode(event)
 	if err != nil {
 		panic(err)
 	}

fvm/environment/event_encoder.go

@@ -2,7 +2,7 @@ package environment
 
 import (
 	"github.com/onflow/cadence"
-	jsoncdc "github.com/onflow/cadence/encoding/json"
+	"github.com/onflow/cadence/encoding/ccf"
 )
 
 type EventEncoder interface {
@@ -16,5 +16,5 @@ func NewCadenceEventEncoder() *CadenceEventEncoder {
 }
 
 func (e *CadenceEventEncoder) Encode(event cadence.Event) ([]byte, error) {
-	return jsoncdc.Encode(event)
+	return ccf.Encode(event)
 }

fvm/fvm_bench_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/onflow/cadence"
+	"github.com/onflow/cadence/encoding/ccf"
 	jsoncdc "github.com/onflow/cadence/encoding/json"
 	"github.com/onflow/cadence/runtime"
 
@@ -308,7 +309,7 @@ func (b *BasicBlockExecutor) SetupAccounts(tb testing.TB, privateKeys []flow.Acc
 
 		for _, event := range computationResult.AllEvents() {
 			if event.Type == flow.EventAccountCreated {
-				data, err := jsoncdc.Decode(nil, event.Payload)
+				data, err := ccf.Decode(nil, event.Payload)
 				if err != nil {
 					tb.Fatal("setup account failed, error decoding events")
 				}

fvm/fvm_blockcontext_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/onflow/cadence"
+	"github.com/onflow/cadence/encoding/ccf"
 	jsoncdc "github.com/onflow/cadence/encoding/json"
 	"github.com/onflow/cadence/runtime"
 	"github.com/stretchr/testify/mock"
@@ -1613,7 +1614,7 @@ func TestBlockContext_GetAccount(t *testing.T) {
 
 		// read the address of the account created (e.g. "0x01" and convert it
 		// to flow.address)
-		data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload)
+		data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload)
 		require.NoError(t, err)
 		address := flow.ConvertAddress(
 			data.(cadence.Event).Fields[0].(cadence.Address))
@@ -1734,7 +1735,7 @@ func TestBlockContext_ExecuteTransaction_CreateAccount_WithMonotonicAddresses(t
 
 	require.Len(t, accountCreatedEvents, 1)
 
-	data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload)
+	data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload)
 	require.NoError(t, err)
 	address := flow.ConvertAddress(
 		data.(cadence.Event).Fields[0].(cadence.Address))

fvm/fvm_fuzz_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/onflow/cadence"
+	"github.com/onflow/cadence/encoding/ccf"
 	jsoncdc "github.com/onflow/cadence/encoding/json"
 
 	"github.com/onflow/flow-go/engine/execution/testutil"
@@ -223,7 +224,7 @@ func getDeductedFees(tb testing.TB, tctx transactionTypeContext, results fuzzRes
 	var feesDeductedEvent cadence.Event
 	for _, e := range results.output.Events {
 		if string(e.Type) == fmt.Sprintf("A.%s.FlowFees.FeesDeducted", environment.FlowFeesAddress(tctx.chain)) {
-			data, err := jsoncdc.Decode(nil, e.Payload)
+			data, err := ccf.Decode(nil, e.Payload)
 			require.NoError(tb, err)
 			feesDeductedEvent, ok = data.(cadence.Event)
 			require.True(tb, ok, "Event payload should be of type cadence event.")
@@ -232,8 +233,15 @@ func getDeductedFees(tb testing.TB, tctx transactionTypeContext, results fuzzRes
 	if feesDeductedEvent.Type() == nil {
 		return 0, false
 	}
-	fees, ok = feesDeductedEvent.Fields[0].(cadence.UFix64)
-	require.True(tb, ok, "FeesDeducted[0] event should be of type cadence.UFix64.")
+
+	for i, f := range feesDeductedEvent.Type().(*cadence.EventType).Fields {
+		if f.Identifier == "amount" {
+			fees, ok = feesDeductedEvent.Fields[i].(cadence.UFix64)
+			require.True(tb, ok, "FeesDeducted event amount field should be of type cadence.UFix64.")
+			break
+		}
+	}
+
 	return fees, true
 }
 
@@ -276,7 +284,7 @@ func bootstrapFuzzStateAndTxContext(tb testing.TB) (bootstrappedVmTest, transact
 		accountCreatedEvents := filterAccountCreatedEvents(output.Events)
 
 		// read the address of the account created (e.g. "0x01" and convert it to flow.address)
-		data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload)
+		data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload)
 		require.NoError(tb, err)
 
 		address = flow.ConvertAddress(

fvm/fvm_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/onflow/cadence"
+	"github.com/onflow/cadence/encoding/ccf"
 	jsoncdc "github.com/onflow/cadence/encoding/json"
 	"github.com/onflow/cadence/runtime"
 	"github.com/onflow/cadence/runtime/common"
@@ -695,16 +696,30 @@ func TestTransactionFeeDeduction(t *testing.T) {
 				unittest.EnsureEventsIndexSeq(t, output.Events, chain.ChainID())
 				require.NotEmpty(t, feeDeduction.Payload)
 
-				payload, err := jsoncdc.Decode(nil, feeDeduction.Payload)
+				payload, err := ccf.Decode(nil, feeDeduction.Payload)
 				require.NoError(t, err)
 
 				event := payload.(cadence.Event)
 
-				require.Equal(t, txFees, event.Fields[0].ToGoValue())
+				var actualTXFees any
+				var actualInclusionEffort any
+				var actualExecutionEffort any
+				for i, f := range event.EventType.Fields {
+					switch f.Identifier {
+					case "amount":
+						actualTXFees = event.Fields[i].ToGoValue()
+					case "executionEffort":
+						actualExecutionEffort = event.Fields[i].ToGoValue()
+					case "inclusionEffort":
+						actualInclusionEffort = event.Fields[i].ToGoValue()
+					}
+				}
+
+				require.Equal(t, txFees, actualTXFees)
 				// Inclusion effort should be equivalent to 1.0 UFix64
-				require.Equal(t, uint64(100_000_000), event.Fields[1].ToGoValue())
+				require.Equal(t, uint64(100_000_000), actualInclusionEffort)
 				// Execution effort should be non-0
-				require.Greater(t, event.Fields[2].ToGoValue(), uint64(0))
+				require.Greater(t, actualExecutionEffort, uint64(0))
 
 			},
 		},
@@ -935,7 +950,7 @@ func TestTransactionFeeDeduction(t *testing.T) {
 			require.Len(t, accountCreatedEvents, 1)
 
 			// read the address of the account created (e.g. "0x01" and convert it to flow.address)
-			data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload)
+			data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload)
 			require.NoError(t, err)
 			address := flow.ConvertAddress(
 				data.(cadence.Event).Fields[0].(cadence.Address))
@@ -1432,12 +1447,21 @@ func TestSettingExecutionWeights(t *testing.T) {
 			for _, event := range output.Events {
 				// the fee deduction event should only contain the max gas worth of execution effort.
 				if strings.Contains(string(event.Type), "FlowFees.FeesDeducted") {
-					ev, err := jsoncdc.Decode(nil, event.Payload)
+					v, err := ccf.Decode(nil, event.Payload)
 					require.NoError(t, err)
+
+					ev := v.(cadence.Event)
+					var actualExecutionEffort any
+					for i, f := range ev.Type().(*cadence.EventType).Fields {
+						if f.Identifier == "executionEffort" {
+							actualExecutionEffort = ev.Fields[i].ToGoValue()
+						}
+					}
+
 					require.Equal(
 						t,
 						maxExecutionEffort,
-						ev.(cadence.Event).Fields[2].ToGoValue().(uint64))
+						actualExecutionEffort)
 				}
 			}
 			unittest.EnsureEventsIndexSeq(t, output.Events, chain.ChainID())
@@ -2085,7 +2109,7 @@ func TestInteractionLimit(t *testing.T) {
 			accountCreatedEvents := filterAccountCreatedEvents(output.Events)
 
 			// read the address of the account created (e.g. "0x01" and convert it to flow.address)
-			data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload)
+			data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload)
 			if err != nil {
 				return snapshotTree, err
 			}