fix: add memos to send-many-memo rosetta STX transfer operations

.github/workflows/ci.yml

@@ -497,6 +497,7 @@ jobs:
         with:
           token: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }}
           fetch-depth: 0
+          persist-credentials: false
 
       - name: Semantic Release
         uses: cycjimmy/semantic-release-action@v2.7.0
@@ -562,11 +563,11 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
+          build-args: |
+            STACKS_API_VERSION=${{ github.head_ref || github.ref_name }}
           file: docker/stx-rosetta.Dockerfile
           tags: ${{ steps.meta_standalone.outputs.tags }}
           labels: ${{ steps.meta_standalone.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
           # Only push if (there's a new release on main branch, or if building a non-main branch) and (Only run on non-PR events or only PRs that aren't from forks)
           push: ${{ (github.ref != 'refs/heads/master' || steps.semantic.outputs.new_release_version != '') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
 

docker/stx-rosetta.Dockerfile

@@ -1,8 +1,8 @@
-ARG STACKS_API_VERSION=v0.71.2
+ARG STACKS_API_VERSION
 ARG STACKS_NODE_VERSION=2.05.0.4.0
-ARG STACKS_API_REPO=blockstack/stacks-blockchain-api
-ARG STACKS_NODE_REPO=blockstack/stacks-blockchain
-ARG PG_VERSION=12
+ARG STACKS_API_REPO=hirosystems/stacks-blockchain-api
+ARG STACKS_NODE_REPO=stacks-network/stacks-blockchain
+ARG PG_VERSION=14
 ARG STACKS_NETWORK=mainnet
 ARG STACKS_LOG_DIR=/var/log/stacks-node
 ARG STACKS_SVC_DIR=/etc/service
@@ -30,7 +30,7 @@ RUN apt-get update -y \
         jq \
         openjdk-11-jre-headless \
         cmake \
-    && git clone -b ${STACKS_API_VERSION} --depth 1 https://github.com/${STACKS_API_REPO} . \
+    && git clone -b ${STACKS_API_VERSION} https://github.com/${STACKS_API_REPO} . \
     && echo "GIT_TAG=$(git tag --points-at HEAD)" >> .env \
     && npm config set unsafe-perm true \
     && npm ci \

src/api/controllers/db-controller.ts

@@ -1,5 +1,6 @@
 import {
   abiFunctionToString,
+  ChainID,
   ClarityAbi,
   ClarityAbiFunction,
   getTypeString,
@@ -288,12 +289,14 @@ export function parseDbEvent(dbEvent: DbEvent): TransactionEvent {
  * If neither argument is present, the most recent block is returned.
  * @param db -- datastore
  * @param fetchTransactions -- return block transactions
+ * @param chainId -- chain ID
  * @param blockHash -- hexadecimal hash string
  * @param blockHeight -- number
  */
 export async function getRosettaBlockFromDataStore(
   db: PgStore,
   fetchTransactions: boolean,
+  chainId: ChainID,
   blockHash?: string,
   blockHeight?: number
 ): Promise<FoundOrNot<RosettaBlock>> {
@@ -318,6 +321,7 @@ export async function getRosettaBlockFromDataStore(
       blockHash: dbBlock.block_hash,
       indexBlockHash: dbBlock.index_block_hash,
       db,
+      chainId,
     });
   }
 
@@ -493,6 +497,7 @@ async function parseRosettaTxDetail(opts: {
   db: PgStore;
   minerRewards: DbMinerReward[];
   unlockingEvents: StxUnlockEvent[];
+  chainId: ChainID;
 }): Promise<RosettaTransaction> {
   let events: DbEvent[] = [];
   if (opts.block_height > 1) {
@@ -508,6 +513,7 @@ async function parseRosettaTxDetail(opts: {
   const operations = await getOperations(
     opts.tx,
     opts.db,
+    opts.chainId,
     opts.minerRewards,
     events,
     opts.unlockingEvents
@@ -529,6 +535,7 @@ async function getRosettaBlockTxFromDataStore(opts: {
   tx: DbTx;
   block: DbBlock;
   db: PgStore;
+  chainId: ChainID;
 }): Promise<FoundOrNot<RosettaTransaction>> {
   let minerRewards: DbMinerReward[] = [],
     unlockingEvents: StxUnlockEvent[] = [];
@@ -545,6 +552,7 @@ async function getRosettaBlockTxFromDataStore(opts: {
     indexBlockHash: opts.tx.index_block_hash,
     tx: opts.tx,
     db: opts.db,
+    chainId: opts.chainId,
     minerRewards,
     unlockingEvents,
   });
@@ -555,6 +563,7 @@ async function getRosettaBlockTransactionsFromDataStore(opts: {
   blockHash: string;
   indexBlockHash: string;
   db: PgStore;
+  chainId: ChainID;
 }): Promise<FoundOrNot<RosettaTransaction[]>> {
   const blockQuery = await opts.db.getBlock({ hash: opts.blockHash });
   if (!blockQuery.found) {
@@ -580,6 +589,7 @@ async function getRosettaBlockTransactionsFromDataStore(opts: {
       indexBlockHash: opts.indexBlockHash,
       tx,
       db: opts.db,
+      chainId: opts.chainId,
       minerRewards,
       unlockingEvents,
     });
@@ -591,7 +601,8 @@ async function getRosettaBlockTransactionsFromDataStore(opts: {
 
 export async function getRosettaTransactionFromDataStore(
   txId: string,
-  db: PgStore
+  db: PgStore,
+  chainId: ChainID
 ): Promise<FoundOrNot<RosettaTransaction>> {
   const txQuery = await db.getTx({ txId, includeUnanchored: false });
   if (!txQuery.found) {
@@ -609,6 +620,7 @@ export async function getRosettaTransactionFromDataStore(
     tx: txQuery.result,
     block: blockQuery.result,
     db,
+    chainId,
   });
 
   if (!rosettaTx.found) {

src/api/routes/rosetta/block.ts

@@ -24,13 +24,13 @@ export function createRosettaBlockRouter(db: PgStore, chainId: ChainID): express
         return;
       }
 
-      let block_hash = req.body.block_identifier?.hash;
-      const index = req.body.block_identifier?.index;
+      let block_hash = req.body.block_identifier?.hash as string | undefined;
+      const index = req.body.block_identifier?.index as number | undefined;
       if (block_hash && !has0xPrefix(block_hash)) {
         block_hash = '0x' + block_hash;
       }
 
-      const block = await getRosettaBlockFromDataStore(db, true, block_hash, index);
+      const block = await getRosettaBlockFromDataStore(db, true, chainId, block_hash, index);
 
       if (!block.found) {
         res.status(500).json(RosettaErrors[RosettaErrorsTypes.blockNotFound]);
@@ -57,7 +57,7 @@ export function createRosettaBlockRouter(db: PgStore, chainId: ChainID): express
         tx_hash = '0x' + tx_hash;
       }
 
-      const transaction = await getRosettaTransactionFromDataStore(tx_hash, db);
+      const transaction = await getRosettaTransactionFromDataStore(tx_hash, db, chainId);
       if (!transaction.found) {
         res.status(500).json(RosettaErrors[RosettaErrorsTypes.transactionNotFound]);
         return;

src/api/routes/rosetta/construction.ts

@@ -514,7 +514,7 @@ export function createRosettaConstructionRouter(db: PgStore, chainId: ChainID):
       }
       try {
         const baseTx = rawTxToBaseTx(inputTx);
-        const operations = await getOperations(baseTx, db);
+        const operations = await getOperations(baseTx, db, chainId);
         const txMemo = parseTransactionMemo(baseTx);
         let response: RosettaConstructionParseResponse;
         if (signed) {

src/api/routes/rosetta/mempool.ts

@@ -69,7 +69,7 @@ export function createRosettaMempoolRouter(db: PgStore, chainId: ChainID): expre
         return;
       }
 
-      const operations = await getOperations(mempoolTxQuery.result, db);
+      const operations = await getOperations(mempoolTxQuery.result, db, chainId);
       const txMemo = parseTransactionMemo(mempoolTxQuery.result);
       const transaction: RosettaTransaction = {
         transaction_identifier: { hash: tx_id },

src/api/routes/rosetta/network.ts

@@ -48,13 +48,13 @@ export function createRosettaNetworkRouter(db: PgStore, chainId: ChainID): expre
         return;
       }
 
-      const block = await getRosettaBlockFromDataStore(db, false);
+      const block = await getRosettaBlockFromDataStore(db, false, chainId);
       if (!block.found) {
         res.status(500).json(RosettaErrors[RosettaErrorsTypes.blockNotFound]);
         return;
       }
 
-      const genesis = await getRosettaBlockFromDataStore(db, false, undefined, 1);
+      const genesis = await getRosettaBlockFromDataStore(db, false, chainId, undefined, 1);
       if (!genesis.found) {
         res.status(500).json(RosettaErrors[RosettaErrorsTypes.blockNotFound]);
         return;

src/rosetta-helpers.ts

@@ -9,10 +9,12 @@ import {
 import {
   addressToString,
   AuthType,
+  BufferCV,
   BufferReader,
   ChainID,
   deserializeTransaction,
   emptyMessageSignature,
+  hexToCV,
   isSingleSig,
   makeSigHashPreSign,
   MessageSignature,
@@ -54,10 +56,10 @@ import {
   StxUnlockEvent,
 } from './datastore/common';
 import { getTxSenderAddress, getTxSponsorAddress } from './event-stream/reader';
-import { unwrapOptional, bufferToHexPrefixString, hexToBuffer, logger } from './helpers';
+import { unwrapOptional, hexToBuffer, logger, getSendManyContract } from './helpers';
 
 import { getCoreNodeEndpoint } from './core-rpc/client';
-import { getBTCAddress, poxAddressToBtcAddress } from '@stacks/stacking';
+import { poxAddressToBtcAddress } from '@stacks/stacking';
 import { TokenMetadataErrorMode } from './token-metadata/tokens-contract-handler';
 import {
   ClarityTypeID,
@@ -71,8 +73,10 @@ import {
   ClarityValueTuple,
   ClarityValueUInt,
   PrincipalTypeID,
-  TxPayloadTokenTransfer,
   TxPayloadTypeID,
+  decodeClarityValueList,
+  ClarityValue,
+  ClarityValueList,
 } from 'stacks-encoding-native-js';
 import { PgStore } from './datastore/pg-store';
 import { isFtMetadataEnabled, tokenMetadataErrorMode } from './token-metadata/helpers';
@@ -124,6 +128,7 @@ export function parseTransactionMemo(tx: BaseTx): string | null {
 export async function getOperations(
   tx: DbTx | DbMempoolTx | BaseTx,
   db: PgStore,
+  chainID: ChainID,
   minerRewards?: DbMinerReward[],
   events?: DbEvent[],
   stxUnlockEvents?: StxUnlockEvent[]
@@ -161,7 +166,7 @@ export async function getOperations(
   }
 
   if (events !== undefined) {
-    await processEvents(db, events, tx, operations);
+    await processEvents(db, events, tx, operations, chainID);
   }
 
   return operations;
@@ -173,12 +178,54 @@ function processUnlockingEvents(events: StxUnlockEvent[], operations: RosettaOpe
   });
 }
 
+/**
+ * If `tx` is a contract call to the `send-many-memo` contract, return an array of `memo` values for
+ * all STX transfers sorted by event index.
+ * @param tx - Base transaction
+ * @returns Array of `memo` values
+ */
+function decodeSendManyContractCallMemos(tx: BaseTx, chainID: ChainID): string[] | undefined {
+  if (
+    getTxTypeString(tx.type_id) === 'contract_call' &&
+    tx.contract_call_contract_id === getSendManyContract(chainID) &&
+    tx.contract_call_function_name &&
+    ['send-many', 'send-stx-with-memo'].includes(tx.contract_call_function_name) &&
+    tx.contract_call_function_args
+  ) {
+    const decodeMemo = (memo?: ClarityValue): string => {
+      return memo && memo.type_id === ClarityTypeID.Buffer
+        ? (hexToCV(memo.hex) as BufferCV).buffer.toString('utf8')
+        : '';
+    };
+    try {
+      const argList = decodeClarityValueList(tx.contract_call_function_args, true);
+      if (tx.contract_call_function_name === 'send-many') {
+        const list = argList[0] as ClarityValueList<ClarityValue>;
+        return (list.list as ClarityValueTuple[]).map(item => decodeMemo(item.data.memo));
+      } else if (tx.contract_call_function_name === 'send-stx-with-memo') {
+        return [decodeMemo(argList[2])];
+      }
+    } catch (error) {
+      logger.warn(`Could not decode send-many-memo arguments: ${error}`);
+      return;
+    }
+  }
+}
+
 async function processEvents(
   db: PgStore,
   events: DbEvent[],
   baseTx: BaseTx,
-  operations: RosettaOperation[]
+  operations: RosettaOperation[],
+  chainID: ChainID
 ) {
+  // Is this a `send-many-memo` contract call transaction? If so, we must include the provided
+  // `memo` values inside STX operation metadata entries. STX transfer events inside
+  // `send-many-memo` contract calls come in the same order as the provided args, therefore we can
+  // match them by index.
+  const sendManyMemos = decodeSendManyContractCallMemos(baseTx, chainID);
+  let sendManyStxTransferEventIndex = 0;
+
   for (const event of events) {
     const txEventType = event.event_type;
     switch (txEventType) {
@@ -205,8 +252,16 @@ async function processEvents(
               stxAssetEvent.amount,
               () => 'Unexpected nullish amount'
             );
-            operations.push(makeSenderOperation(tx, operations.length));
-            operations.push(makeReceiverOperation(tx, operations.length));
+            let index = operations.length;
+            const sender = makeSenderOperation(tx, index++);
+            const receiver = makeReceiverOperation(tx, index++);
+            if (sendManyMemos) {
+              sender.metadata = receiver.metadata = {
+                memo: sendManyMemos[sendManyStxTransferEventIndex++],
+              };
+            }
+            operations.push(sender);
+            operations.push(receiver);
             break;
           case DbAssetEventTypeId.Burn:
             operations.push(makeBurnOperation(stxAssetEvent, baseTx, operations.length));
@@ -1009,7 +1064,7 @@ export function rawTxToBaseTx(raw_tx: string): BaseTx {
       transactionType = DbTxTypeId.PoisonMicroblock;
       break;
   }
-  const dbtx: BaseTx = {
+  const dbTx: BaseTx = {
     token_transfer_recipient_address: recipientAddr,
     tx_id: txId,
     anchor_mode: 3,
@@ -1025,10 +1080,10 @@ export function rawTxToBaseTx(raw_tx: string): BaseTx {
 
   const txPayload = transaction.payload;
   if (txPayload.type_id === TxPayloadTypeID.TokenTransfer) {
-    dbtx.token_transfer_memo = txPayload.memo_hex;
+    dbTx.token_transfer_memo = txPayload.memo_hex;
   }
 
-  return dbtx;
+  return dbTx;
 }
 
 export async function getValidatedFtMetadata(

src/test-utils/test-builders.ts

@@ -394,6 +394,9 @@ interface TestSmartContractLogEventArgs {
   contract_identifier?: string;
   event_index?: number;
   tx_index?: number;
+  canonical?: boolean;
+  topic?: string;
+  value?: string;
 }
 
 /**
@@ -407,11 +410,11 @@ function testSmartContractLogEvent(args?: TestSmartContractLogEventArgs): DbSmar
     tx_id: args?.tx_id ?? TX_ID,
     tx_index: args?.tx_index ?? 0,
     block_height: args?.block_height ?? BLOCK_HEIGHT,
-    canonical: true,
+    canonical: args?.canonical ?? true,
     event_type: DbEventTypeId.SmartContractLog,
     contract_identifier: args?.contract_identifier ?? CONTRACT_ID,
-    topic: 'some-topic',
-    value: bufferToHexPrefixString(serializeCV(bufferCVFromString('some val'))),
+    topic: args?.topic ?? 'some-topic',
+    value: args?.value ?? bufferToHexPrefixString(serializeCV(bufferCVFromString('some val'))),
   };
 }