Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

integration-tests/deployment/ccip: fix assertion fns #14482

Merged
merged 5 commits into from
Sep 18, 2024
Merged

integration-tests/deployment/ccip: fix assertion fns #14482

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7344b9f
integration-tests/deployment/ccip: fix assertion fns
makramkd Sep 18, 2024
1a67785
fix stringer
makramkd Sep 18, 2024
cacaacd
Merge branch 'develop' into mk/fix-flakey-ccip-test
makramkd Sep 18, 2024
28cf284
use getExecutionState to assert test condition
makramkd Sep 18, 2024
cb2ea3c
Merge branch 'develop' into mk/fix-flakey-ccip-test
makramkd Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 64 additions & 25 deletions integration-tests/deployment/ccip/test_assertions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,20 @@ func ConfirmCommitForAllWithExpectedSeqNums(
srcChain := srcChain
dstChain := dstChain
wg.Go(func() error {
return func(src, dest uint64) error {
var startBlock *uint64
if startBlocks != nil {
startBlock = startBlocks[dest]
}
return ConfirmCommitWithExpectedSeqNumRange(t, srcChain, dstChain, state.Chains[dest].OffRamp, startBlock,
ccipocr3.SeqNumRange{ccipocr3.SeqNum(expectedSeqNums[dest]), ccipocr3.SeqNum(expectedSeqNums[dest])})
}(src, dest)
var startBlock *uint64
if startBlocks != nil {
startBlock = startBlocks[dstChain.Selector]
}
return ConfirmCommitWithExpectedSeqNumRange(
t,
srcChain,
dstChain,
state.Chains[dstChain.Selector].OffRamp,
startBlock,
ccipocr3.SeqNumRange{
ccipocr3.SeqNum(expectedSeqNums[dstChain.Selector]),
ccipocr3.SeqNum(expectedSeqNums[dstChain.Selector]),
Comment on lines +51 to +52
Copy link
Contributor

@winder winder Sep 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Input should probably be expectedSeqNums map[uint64]SeqNumRange since we'll need that eventually.

})
})
}
}
Expand Down Expand Up @@ -131,16 +137,18 @@ func ConfirmExecWithSeqNrForAll(
srcChain := srcChain
dstChain := dstChain
wg.Go(func() error {
return func(src, dest deployment.Chain) error {
var startBlock *uint64
if startBlocks != nil {
startBlock = startBlocks[dest.Selector]
}
return ConfirmExecWithSeqNr(
t, src, dest, state.Chains[dest.Selector].OffRamp, startBlock,
expectedSeqNums[dstChain.Selector],
)
}(srcChain, dstChain)
var startBlock *uint64
if startBlocks != nil {
startBlock = startBlocks[dstChain.Selector]
}
return ConfirmExecWithSeqNr(
t,
srcChain,
dstChain,
state.Chains[dstChain.Selector].OffRamp,
startBlock,
expectedSeqNums[dstChain.Selector],
)
})
}
}
Expand Down Expand Up @@ -185,19 +193,50 @@ func ConfirmExecWithSeqNr(
if err != nil {
return fmt.Errorf("error to get source chain config : %w", err)
}
t.Logf("Waiting for ExecutionStateChanged on chain %d from chain %d with expected sequence number %d, current onchain minSeqNr: %d",
dest.Selector, source.Selector, expectedSeqNr, scc.MinSeqNr)
executionState, err := offRamp.GetExecutionState(nil, source.Selector, expectedSeqNr)
if err != nil {
return fmt.Errorf("error to get execution state : %w", err)
}
t.Logf("Waiting for ExecutionStateChanged on chain %d (offramp %s) from chain %d with expected sequence number %d, current onchain minSeqNr: %d, execution state: %s",
dest.Selector, offRamp.Address().String(), source.Selector, expectedSeqNr, scc.MinSeqNr, executionStateToString(executionState))
Comment on lines +200 to +201
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Adding these logs to aid in test debugging - in case the log fetching is (for whatever reason) not working.

if executionState == EXECUTION_STATE_SUCCESS {
t.Logf("Observed SUCCESS execution state on chain %d (offramp %s) from chain %d with expected sequence number %d",
dest.Selector, offRamp.Address().String(), source.Selector, expectedSeqNr)
return nil
}
Comment on lines +202 to +206
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Adding this because in the flakes I observed the following phenomenon:

2024-09-18T16:35:18.1257108Z     Waiting for ExecutionStateChanged on chain 5548718428018410741 (offramp 0x5E6f8839581AfeE68f899bF33B8F284FFf92ef0E) from chain 909606746561742123 with expected sequence number 1, current onchain minSeqNr: 2, execution state: UNTOUCHED
2024-09-18T16:35:18.1258349Z     Waiting for ExecutionStateChanged on chain 909606746561742123 (offramp 0xBAa5D0009A5a445ab05F190923dBFd0624BC9dd7) from chain 5548718428018410741 with expected sequence number 1, current onchain minSeqNr: 2, execution state: UNTOUCHED
2024-09-18T16:35:18.1259586Z     Waiting for ExecutionStateChanged on chain 5548718428018410741 (offramp 0x5E6f8839581AfeE68f899bF33B8F284FFf92ef0E) from chain 789068866484373046 with expected sequence number 1, current onchain minSeqNr: 2, execution state: SUCCESS
2024-09-18T16:35:18.1260817Z     Waiting for ExecutionStateChanged on chain 909606746561742123 (offramp 0xBAa5D0009A5a445ab05F190923dBFd0624BC9dd7) from chain 789068866484373046 with expected sequence number 1, current onchain minSeqNr: 2, execution state: SUCCESS
2024-09-18T16:35:18.1262055Z     Waiting for ExecutionStateChanged on chain 789068866484373046 (offramp 0xAF84A0E10C914dD6d91c2e1674031f4668Ada3FB) from chain 909606746561742123 with expected sequence number 1, current onchain minSeqNr: 2, execution state: UNTOUCHED
2024-09-18T16:35:18.1263291Z     Waiting for ExecutionStateChanged on chain 789068866484373046 (offramp 0xAF84A0E10C914dD6d91c2e1674031f4668Ada3FB) from chain 5548718428018410741 with expected sequence number 1, current onchain minSeqNr: 2, execution state: UNTOUCHED

i.e even before subscribing it seems that we successfully executed some messages. The subscription never ended up picking up these logs and relaying them to the channel. So this is hopefully going to aid in those cases.

case execEvent := <-sink:
if execEvent.SequenceNumber == expectedSeqNr && execEvent.SourceChainSelector == source.Selector {
t.Logf("Received ExecutionStateChanged on chain %d from chain %d with expected sequence number %d",
dest.Selector, source.Selector, expectedSeqNr)
t.Logf("Received ExecutionStateChanged on chain %d (offramp %s) from chain %d with expected sequence number %d",
dest.Selector, offRamp.Address().String(), source.Selector, expectedSeqNr)
return nil
}
case <-timer.C:
return fmt.Errorf("timed out waiting for ExecutionStateChanged on chain %d from chain %d with expected sequence number %d",
dest.Selector, source.Selector, expectedSeqNr)
return fmt.Errorf("timed out waiting for ExecutionStateChanged on chain %d (offramp %s) from chain %d with expected sequence number %d",
dest.Selector, offRamp.Address().String(), source.Selector, expectedSeqNr)
case subErr := <-subscription.Err():
return fmt.Errorf("Subscription error: %w", subErr)
return fmt.Errorf("subscription error: %w", subErr)
}
}
}

const (
EXECUTION_STATE_UNTOUCHED = 0
EXECUTION_STATE_INPROGRESS = 1
EXECUTION_STATE_SUCCESS = 2
EXECUTION_STATE_FAILURE = 3
)

func executionStateToString(state uint8) string {
switch state {
case EXECUTION_STATE_UNTOUCHED:
return "UNTOUCHED"
case EXECUTION_STATE_INPROGRESS:
return "IN_PROGRESS"
case EXECUTION_STATE_SUCCESS:
return "SUCCESS"
case EXECUTION_STATE_FAILURE:
return "FAILURE"
default:
return "UNKNOWN"
}
}
Loading