add pfm testing scripts

Makefile

@@ -70,6 +70,7 @@ BUILD_FLAGS := -tags "$(build_tags)" -ldflags '$(ldflags)'
 ###############################################################################
 
 build:
+	which go
 	mkdir -p $(BUILDDIR)/
 	go build -mod=readonly $(BUILD_FLAGS) -trimpath -o $(BUILDDIR) ./...;
 

app/app.go

@@ -157,6 +157,9 @@ import (
 	ccvstaking "github.com/cosmos/interchain-security/v3/x/ccv/democracy/staking"
 
 	storetypes "github.com/cosmos/cosmos-sdk/store/types"
+	"github.com/cosmos/ibc-apps/middleware/packet-forward-middleware/v7/packetforward"
+	packetforwardkeeper "github.com/cosmos/ibc-apps/middleware/packet-forward-middleware/v7/packetforward/keeper"
+	packetforwardtypes "github.com/cosmos/ibc-apps/middleware/packet-forward-middleware/v7/packetforward/types"
 )
 
 const (
@@ -225,6 +228,7 @@ var (
 		autopilot.AppModuleBasic{},
 		icaoracle.AppModuleBasic{},
 		tendermint.AppModuleBasic{},
+		packetforward.AppModuleBasic{},
 		evmosvesting.AppModuleBasic{},
 	)
 
@@ -302,6 +306,7 @@ type StrideApp struct {
 	ICAHostKeeper       icahostkeeper.Keeper
 	ConsumerKeeper      ccvconsumerkeeper.Keeper
 	AutopilotKeeper     autopilotkeeper.Keeper
+	PacketForwardKeeper *packetforwardkeeper.Keeper
 
 	// make scoped keepers public for test purposes
 	ScopedIBCKeeper      capabilitykeeper.ScopedKeeper
@@ -369,6 +374,7 @@ func NewStrideApp(
 		ccvconsumertypes.StoreKey,
 		crisistypes.StoreKey,
 		consensusparamtypes.StoreKey,
+		packetforwardtypes.StoreKey,
 		evmosvestingtypes.StoreKey,
 	)
 	tkeys := sdk.NewTransientStoreKeys(paramstypes.TStoreKey)
@@ -473,22 +479,38 @@ func NewStrideApp(
 		app.GetSubspace(ratelimitmoduletypes.ModuleName),
 		app.BankKeeper,
 		app.IBCKeeper.ChannelKeeper,
-		// TODO: Implement ICS4Wrapper in Records and pass records keeper here
 		app.IBCKeeper.ChannelKeeper, // ICS4Wrapper
 	)
 	ratelimitModule := ratelimitmodule.NewAppModule(appCodec, app.RatelimitKeeper)
 
+	// Initialize the packet forward middleware Keeper
+	// It's important to note that the PFM Keeper must be initialized before the Transfer Keeper
+	app.PacketForwardKeeper = packetforwardkeeper.NewKeeper(
+		appCodec,
+		keys[packetforwardtypes.StoreKey],
+		nil, // will be zero-value here, reference is set later on with SetTransferKeeper.
+		app.IBCKeeper.ChannelKeeper,
+		app.DistrKeeper,
+		app.BankKeeper,
+		app.RatelimitKeeper, // ICS4Wrapper
+		authtypes.NewModuleAddress(govtypes.ModuleName).String(),
+	)
+
 	// Create Transfer Keepers
 	app.TransferKeeper = ibctransferkeeper.NewKeeper(
 		appCodec, keys[ibctransfertypes.StoreKey],
 		app.GetSubspace(ibctransfertypes.ModuleName),
-		app.RatelimitKeeper, // ICS4Wrapper
+		app.PacketForwardKeeper, // ICS4Wrapper
 		app.IBCKeeper.ChannelKeeper,
 		&app.IBCKeeper.PortKeeper,
 		app.AccountKeeper,
 		app.BankKeeper,
 		scopedTransferKeeper,
 	)
+
+	// Set the TransferKeeper reference in the PacketForwardKeeper
+	app.PacketForwardKeeper.SetTransferKeeper(app.TransferKeeper)
+
 	transferModule := transfer.NewAppModule(app.TransferKeeper)
 	transferIBCModule := transfer.NewIBCModule(app.TransferKeeper)
 
@@ -703,14 +725,27 @@ func NewStrideApp(
 	icacallbacksStack = icaoracle.NewIBCMiddleware(icacallbacksStack, app.ICAOracleKeeper)
 	icacallbacksStack = icacontroller.NewIBCMiddleware(icacallbacksStack, app.ICAControllerKeeper)
 
+	// SendPacket originates from the base app and work up the stack to core IBC
+	// RecvPacket originates from core IBC and goes down the stack
+
 	// Stack three contains
-	// - IBC
+	// - core ibc
 	// - autopilot
 	// - records
 	// - ratelimit
+	// - pfm
 	// - transfer
 	// - base app
+	// Note: Traffic up the stack does not pass through records or autopilot,
+	// as defined via the ICS4Wrappers of each keeper
 	var transferStack porttypes.IBCModule = transferIBCModule
+	transferStack = packetforward.NewIBCMiddleware(
+		transferStack,
+		app.PacketForwardKeeper,
+		0, // retries on timeout
+		packetforwardkeeper.DefaultForwardTransferPacketTimeoutTimestamp, // forward timeout
+		packetforwardkeeper.DefaultRefundTransferPacketTimeoutTimestamp,  // refund timeout
+	)
 	transferStack = ratelimitmodule.NewIBCMiddleware(app.RatelimitKeeper, transferStack)
 	transferStack = recordsmodule.NewIBCModule(app.RecordsKeeper, transferStack)
 	transferStack = autopilot.NewIBCModule(app.AutopilotKeeper, transferStack)
@@ -762,6 +797,8 @@ func NewStrideApp(
 		ibc.NewAppModule(app.IBCKeeper),
 		params.NewAppModule(app.ParamsKeeper),
 		claim.NewAppModule(appCodec, app.ClaimKeeper),
+		// technically, app.GetSubspace(packetforwardtypes.ModuleName) will never be run https://github.com/cosmos/ibc-apps/issues/146#issuecomment-1839242144
+		packetforward.NewAppModule(app.PacketForwardKeeper, app.GetSubspace(packetforwardtypes.ModuleName)),
 		transferModule,
 		// monitoringModule,
 		stakeibcModule,
@@ -813,6 +850,7 @@ func NewStrideApp(
 		autopilottypes.ModuleName,
 		icaoracletypes.ModuleName,
 		consensusparamtypes.ModuleName,
+		packetforwardtypes.ModuleName,
 	)
 
 	app.mm.SetOrderEndBlockers(
@@ -848,6 +886,7 @@ func NewStrideApp(
 		autopilottypes.ModuleName,
 		icaoracletypes.ModuleName,
 		consensusparamtypes.ModuleName,
+		packetforwardtypes.ModuleName,
 	)
 
 	// NOTE: The genutils module must occur after staking so that pools are
@@ -888,6 +927,7 @@ func NewStrideApp(
 		autopilottypes.ModuleName,
 		icaoracletypes.ModuleName,
 		consensusparamtypes.ModuleName,
+		packetforwardtypes.ModuleName,
 	)
 
 	app.mm.RegisterInvariants(app.CrisisKeeper)
@@ -1160,6 +1200,7 @@ func initParamsKeeper(appCodec codec.BinaryCodec, legacyAmino *codec.LegacyAmino
 	paramsKeeper.Subspace(icacallbacksmoduletypes.ModuleName)
 	paramsKeeper.Subspace(ccvconsumertypes.ModuleName)
 	paramsKeeper.Subspace(autopilottypes.ModuleName)
+	paramsKeeper.Subspace(packetforwardtypes.ModuleName).WithKeyTable(packetforwardtypes.ParamKeyTable())
 	paramsKeeper.Subspace(icaoracletypes.ModuleName)
 	paramsKeeper.Subspace(claimtypes.ModuleName)
 	return paramsKeeper

app/upgrades.go

@@ -257,6 +257,12 @@ func (app *StrideApp) setupUpgradeHandlers(appOpts servertypes.AppOptions) {
 			Added: []string{evmosvestingtypes.ModuleName},
 		}
 	}
+	// TODO: uncomment when v17 upgrade is ready
+	// case "v17":
+	// 	storeUpgrades = &storetypes.StoreUpgrades{
+	// 		// Add PFM store key
+	// 		Added: []string{packetforwardtypes.ModuleName},
+	// 	}
 
 	if storeUpgrades != nil {
 		app.SetStoreLoader(upgradetypes.UpgradeStoreLoader(upgradeInfo.Height, storeUpgrades))

dockernet/scripts/autopilot/liquid_stake_and_forward_chainC_failure_case.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source ${SCRIPT_DIR}/../../config.sh
+
+OSMO_ADDRESS="osmo1uk4ze0x4nvh4fk0xm4jdud58eqn4yxhrqyeqwq"
+DENOM_ATOM_ON_STRIDE="ibc/27394FB092D2ECCD56123C74F36E4C1F926001CEADA9CA97EA622B25F41E5EB2"
+DENOM_ATOM_ON_OSMOSIS_THRU_STRIDE="ibc/6CDD4663F2F09CD62285E2D45891FC149A3568E316CE3EBBE201A71A78A69388"
+
+# VERIFY AUTOPILOT LIQUID STAKING ATOM FROM GAIA WORKS
+memo='{ "autopilot": { "receiver": "'"$(STRIDE_ADDRESS)"'", "stakeibc": { "action": "LiquidStake", "ibc_receiver": "'$(GAIA_ADDRESS)'" } } }'
+$GAIA_MAIN_CMD tx ibc-transfer transfer transfer channel-0 "$memo" 1uatom --from ${GAIA_VAL_PREFIX}1 -y
+
+sleep 10
+
+# NOW TRY AUTOPILOT LIQUID STAKING ATOM FROM OSMOSIS (AFTER THAT ATOM WAS IBC'D THROUGH STRIDE).
+# THIS SHOULD FAIL
+# You should see this error in dockernet's `stride.log`
+    # dockernet-stride1-1  | 1:25AM ERR Error liquid staking packet from autopilot for $OSMO_ADDRESS: the native token is not supported for liquid staking module=x/autopilot
+
+## IBC ATOM from GAIA to STRIDE
+$GAIA_MAIN_CMD tx ibc-transfer transfer transfer channel-0 $(STRIDE_ADDRESS) 1000000uatom --from ${GAIA_VAL_PREFIX}1 -y 
+sleep 10
+$STRIDE_MAIN_CMD q bank balances $(STRIDE_ADDRESS)
+## IBC ATOM from STRIDE to OSMOSIS
+$STRIDE_MAIN_CMD tx ibc-transfer transfer transfer channel-1 $OSMO_ADDRESS "100$ATOM_ON_STRIDE_DENOM" --from ${STRIDE_VAL_PREFIX}1 -y 
+sleep 10
+$OSMO_MAIN_CMD q bank balances $OSMO_ADDRESS
+
+memo='{ "autopilot": { "receiver": "'"$(STRIDE_ADDRESS)"'", "stakeibc": { "action": "LiquidStake", "ibc_receiver": "'$(GAIA_ADDRESS)'" } } }'
+$OSMO_MAIN_CMD tx ibc-transfer transfer transfer channel-0 "$memo" "1$DENOM_ATOM_ON_OSMOSIS_THRU_STRIDE" --from ${OSMO_VAL_PREFIX}1 -y --gas 900000 --gas-prices 0.0025uosmo
+

dockernet/scripts/pfm/pfm_abc_default.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -eu 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source ${SCRIPT_DIR}/../../config.sh
+
+###############################################
+#   Minimal Example - Chain forward A->B->C   #
+###############################################
+# docs: https://github.com/cosmos/ibc-apps/tree/main/middleware/packet-forward-middleware#minimal-example---chain-forward-a-b-c
+
+# to verify pfm works:
+# - open two terminal windows as described below
+    # - terminal 1 monitors the sender on A: `while true; do gaiadl q bank balances $(gaiadl keys show gval1 -a)  && echo "--------------" && sleep 1; done`
+    # - terminal 2 monitors the receiver on C: `while true; do osmosisdl q bank balances osmo1uk4ze0x4nvh4fk0xm4jdud58eqn4yxhrqyeqwq  && echo "--------------" && sleep 1; done`
+# - run `sh dockernet/scripts/pfm/pfm_abc_default.sh`
+# - watch the terminal windows to see C's balance increase after A's descreases.
+
+FINAL_RECEIVER=$(OSMO_ADDRESS)
+CHANNEL_B_TO_C="channel-1"
+INVALID_RECEIVER_CHAINB="pfm" # purposely using invalid bech32 here
+TIMEOUT="10m"
+RETRIES="2"
+
+CHANNEL_BETWEEN_A_AND_B="channel-0"
+
+# note that the memo must be escaped as follows to work from this script.
+memo="{\"forward\":{\"receiver\":\"$FINAL_RECEIVER\",\"port\":\"transfer\",\"channel\":\"$CHANNEL_B_TO_C\",\"timeout\":\"$TIMEOUT\",\"retries\":$RETRIES}}"
+echo $memo
+
+# pfm transfer
+$GAIA_MAIN_CMD tx ibc-transfer transfer transfer $CHANNEL_A_TO_B "pfm" 1uatom --memo "$memo" --from ${GAIA_VAL_PREFIX}1 -y

dockernet/scripts/pfm/pfm_to_autopilot_ls.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -eu 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source ${SCRIPT_DIR}/../../config.sh
+
+########################################################################################
+#     PFM (a -> b -> autopilot-LS-on-a: stride -> osmosis -> stride (autopilot LS)     #
+########################################################################################
+# docs: https://github.com/cosmos/ibc-apps/tree/main/middleware/packet-forward-middleware#full-example---chain-forward-a-b-c-d-with-retry-on-timeout
+
+# to verify pfm works:
+# - open two terminal windows as described below:
+    # - terminal 1 monitors the sender on stride: `while true; do stridedl q bank balances stride1uk4ze0x4nvh4fk0xm4jdud58eqn4yxhrt52vv7 && echo "--------------" && sleep 1; done`
+    # - terminal 2 monitors the final receiver on gaia: `while true; do gaiadl q bank balances cosmos1uk4ze0x4nvh4fk0xm4jdud58eqn4yxhrgl2scj && echo "--------------" && sleep 1; done`
+# - watch the terminal windows to see the receiver's stuatom balance increase after the sender's ibc/atom bal decreases.
+# - run `sh dockernet/scripts/pfm/pfm_to_autopilot_ls.sh`
+
+## Prep: fund account by ibc'ing atom from gaia to stride
+$GAIA_MAIN_CMD tx ibc-transfer transfer transfer channel-0 $(STRIDE_ADDRESS) 100uatom --from ${GAIA_VAL_PREFIX}1 -y 
+sleep 10
+$STRIDE_MAIN_CMD q bank balances $(STRIDE_ADDRESS)
+
+
+STRIDE_SENDER=$($STRIDE_MAIN_CMD keys show ${STRIDE_VAL_PREFIX}1 --keyring-backend test -a | grep $STRIDE_ADDRESS_PREFIX)
+STRIDE_LIQUID_STAKER=$($STRIDE_MAIN_CMD keys show ${STRIDE_VAL_PREFIX}2 --keyring-backend test -a | grep $STRIDE_ADDRESS_PREFIX)
+GAIA_RECEIVER=$(GAIA_ADDRESS)
+echo $STRIDE_SENDER, $STRIDE_LIQUID_STAKER, $GAIA_RECEIVER
+
+CHANNEL_A_TO_B="channel-1" # channel 1: stride -> osmosis
+CHANNEL_B_TO_C="channel-0" # channel 0: osmosis -> stride
+INVALID_RECEIVER_CHAINB="pfm" # purposely using invalid bech32 here
+TIMEOUT="10m"
+RETRIES="2"
+
+autopilot_memo="{\"autopilot\":{\"receiver\":\"$STRIDE_LIQUID_STAKER\",\"stakeibc\":{\"action\":\"LiquidStake\",\"ibc_receiver\":\"$GAIA_RECEIVER\"}}}"
+
+memo="{\"forward\":{\"receiver\":\"$STRIDE_LIQUID_STAKER\",\"port\":\"transfer\",\"channel\":\"$CHANNEL_B_TO_C\",\"timeout\":\"$TIMEOUT\",\"retries\":$RETRIES,\"next\":\"$autopilot_memo\"}}"
+
+echo $memo
+
+# pfm transfer
+IBC_VOUCHER_ATOM_ON_STRIDE="ibc/27394FB092D2ECCD56123C74F36E4C1F926001CEADA9CA97EA622B25F41E5EB2"
+$STRIDE_MAIN_CMD tx ibc-transfer transfer transfer $CHANNEL_A_TO_B "pfm" 1$IBC_VOUCHER_ATOM_ON_STRIDE --memo "$memo" --from ${STRIDE_VAL_PREFIX}1 -y