txpool: execute maintenance on every block (PoC)

.gitlab/pipeline/zombienet/cumulus.yml

@@ -30,6 +30,7 @@
     GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet/tests"
     LOCAL_DIR: "/builds/parity/mirrors/polkadot-sdk/cumulus/zombienet/tests"
     COL_IMAGE: "docker.io/paritypr/test-parachain:${DOCKER_IMAGES_VERSION}"
+    POLKADOT_PARACHIN_IMAGE: "docker.io/paritypr/polkadot-parachain-debug:${DOCKER_IMAGES_VERSION}"
     FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
     RUN_IN_CONTAINER: "1"
   artifacts:
@@ -142,3 +143,16 @@ zombienet-cumulus-0007-full_node_warp_sync:
       --local-dir="${LOCAL_DIR}"
       --concurrency=1
       --test="0007-full_node_warp_sync.zndsl"
+
+
+zombienet-cumulus-0008-parachain_extrinsic_gets_finalized:
+  extends:
+    - .zombienet-cumulus-common
+    - .zombienet-refs
+    - .zombienet-before-script
+    - .zombienet-after-script
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
+      --local-dir="${LOCAL_DIR}"
+      --concurrency=1
+      --test="0008-parachain_extrinsic_gets_finalized.zndsl"

cumulus/zombienet/tests/0008-main.js

@@ -0,0 +1,18 @@
+// Allows to manually submit extrinsic to collator.
+// Usage:
+// zombienet-linux -p native spwan 0008-parachain-extrinsic-gets-finalized.toml
+// node 0008-main.js <path-to-generated-tmpdir/zombie.json>
+
+global.zombie = null
+
+const fs = require('fs');
+const test = require('./0008-transaction_gets_finalized.js');
+
+if (process.argv.length == 2) {
+    console.error('Path to zombie.json (generated by zombienet-linux spawn command shall be given)!');
+    process.exit(1);
+}
+
+let networkInfo = JSON.parse(fs.readFileSync(process.argv[2]));
+
+test.run("charlie", networkInfo).then(process.exit)

cumulus/zombienet/tests/0008-parachain_extrinsic_gets_finalized.toml

@@ -0,0 +1,25 @@
+[relaychain]
+default_image = "{{RELAY_IMAGE}}"
+default_command = "polkadot"
+chain = "rococo-local"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true
+
+[[parachains]]
+id = 2000
+cumulus_based = true
+chain = "asset-hub-kusama-local"
+
+  # run charlie as parachain collator
+  [[parachains.collators]]
+  name = "charlie"
+  validator = true
+  image = "{{POLKADOT_PARACHAIN_IMAGE}}"
+  command = "polkadot-parachain"
+  args = ["--force-authoring"]

cumulus/zombienet/tests/0008-parachain_extrinsic_gets_finalized.zndsl

@@ -0,0 +1,20 @@
+Description: Block building
+Network: ./0008-parachain_extrinsic_gets_finalized.toml
+Creds: config
+
+alice: reports node_roles is 4
+bob: reports node_roles is 4
+charlie: reports node_roles is 4
+
+alice: reports peers count is at least 1
+bob: reports peers count is at least 1
+
+alice: reports block height is at least 5 within 60 seconds
+bob: reports block height is at least 5 within 60 seconds
+charlie: reports block height is at least 2 within 120 seconds
+
+alice: count of log lines containing "error" is 0 within 2 seconds
+bob: count of log lines containing "error" is 0 within 2 seconds
+charlie: count of log lines containing "error" is 0 within 2 seconds
+
+charlie: js-script ./0008-transaction_gets_finalized.js within 60 seconds

cumulus/zombienet/tests/0008-transaction_gets_finalized.js

@@ -0,0 +1,70 @@
+//based on: https://polkadot.js.org/docs/api/examples/promise/transfer-events
+
+const testKeyring = require('@polkadot/keyring/testing');
+const { WsProvider, ApiPromise } = require('@polkadot/api');
+const assert = require("assert");
+require('console-stamp')(console, '[HH:MM:ss.l]');
+
+async function run(nodeName, networkInfo, args) {
+  const {wsUri, userDefinedTypes} = networkInfo.nodesByName[nodeName];
+  // Create the API and wait until ready
+  var api = null;
+  var keyring = null;
+  if (zombie == null) {
+    const provider = new WsProvider(wsUri);
+    api = await ApiPromise.create({provider});
+    // Construct the keyring after the API (crypto has an async init)
+    keyring = testKeyring.createTestKeyring({ type: "sr25519" });
+  } else {
+    keyring = new zombie.Keyring({ type: "sr25519" });
+    api = await zombie.connect(wsUri, userDefinedTypes);
+  }
+
+
+  // Add Alice to our keyring with a hard-derivation path (empty phrase, so uses dev)
+  const alice = keyring.addFromUri('//Alice');
+
+  // Create a extrinsic: 
+  const extrinsic = api.tx.system.remark("xxx");
+
+  let extrinsic_success_event = false;
+  try {
+    await new Promise( async (resolve, reject) => {
+      const unsubscribe = await extrinsic
+        .signAndSend(alice, { nonce: -1 }, ({ events = [], status }) => {
+          console.log('Extrinsic status:', status.type);
+
+          if (status.isInBlock) {
+            console.log('Included at block hash', status.asInBlock.toHex());
+            console.log('Events:');
+
+            events.forEach(({ event: { data, method, section }, phase }) => {
+              console.log('\t', phase.toString(), `: ${section}.${method}`, data.toString());
+
+              if (section=="system" && method =="ExtrinsicSuccess") {
+                extrinsic_success_event = true;
+              }
+            });
+          } else if (status.isFinalized) {
+            console.log('Finalized block hash', status.asFinalized.toHex());
+            unsubscribe();
+            if (extrinsic_success_event) {
+              resolve();
+            } else {
+              reject("ExtrinsicSuccess has not been seen");
+            }
+          } else if (status.isError) {
+            unsubscribe();
+            reject("Extrinsic status.isError");
+          }
+
+        });
+    });
+  } catch (error) {
+    assert.fail("Transfer promise failed, error: " + error);
+  }
+
+  assert.ok("test passed");
+}
+
+module.exports = { run }

substrate/client/api/src/client.rs

@@ -388,7 +388,7 @@ impl<B: BlockT> TryFrom<BlockImportNotification<B>> for ChainEvent<B> {
 		if n.is_new_best {
 			Ok(Self::NewBestBlock { hash: n.hash, tree_route: n.tree_route })
 		} else {
-			Err(())
+			Ok(Self::NewBlock { hash: n.hash, tree_route: n.tree_route })
 		}
 	}
 }

substrate/client/transaction-pool/api/src/lib.rs

@@ -291,6 +291,15 @@ impl<T> ReadyTransactions for std::iter::Empty<T> {
 
 /// Events that the transaction pool listens for.
 pub enum ChainEvent<B: BlockT> {
+	/// New block have been added to the chain.
+	NewBlock {
+		/// Hash of the block.
+		hash: B::Hash,
+		/// Tree route from old best to new best parent that was calculated on import.
+		///
+		/// If `None`, no re-org happened on import.
+		tree_route: Option<Arc<sp_blockchain::TreeRoute<B>>>,
+	},
 	/// New best block have been added to the chain.
 	NewBestBlock {
 		/// Hash of the block.
@@ -313,7 +322,9 @@ impl<B: BlockT> ChainEvent<B> {
 	/// Returns the block hash associated to the event.
 	pub fn hash(&self) -> B::Hash {
 		match self {
-			Self::NewBestBlock { hash, .. } | Self::Finalized { hash, .. } => *hash,
+			Self::NewBlock { hash, .. } |
+			Self::NewBestBlock { hash, .. } |
+			Self::Finalized { hash, .. } => *hash,
 		}
 	}
 

substrate/client/transaction-pool/src/enactment_state.rs

@@ -177,7 +177,8 @@ where
 	/// fallback when tree_route cannot be computed.
 	pub fn force_update(&mut self, event: &ChainEvent<Block>) {
 		match event {
-			ChainEvent::NewBestBlock { hash, .. } => self.recent_best_block = *hash,
+			ChainEvent::NewBlock { hash, .. } | ChainEvent::NewBestBlock { hash, .. } =>
+				self.recent_best_block = *hash,
 			ChainEvent::Finalized { hash, .. } => self.recent_finalized_block = *hash,
 		};
 		log::debug!(

substrate/client/transaction-pool/src/lib.rs

@@ -789,7 +789,7 @@ where
 	Pool: MaintainedTransactionPool<Block = Block>,
 {
 	let import_stream = client
-		.import_notification_stream()
+		.every_import_notification_stream()
 		.filter_map(|n| ready(n.try_into().ok()))
 		.fuse();
 	let finality_stream = client.finality_notification_stream().map(Into::into).fuse();