From e7eb5e63fe6aebdada07e6b68ad0e8f40f8a1119 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Sun, 5 May 2024 02:37:33 +0100 Subject: [PATCH] [GraphQL/MovePackage] Paginate by checkpoint ## Description Adds a query, `Query.packages` for fetching all packages that were introduced within a given checkpoint range. Useful for fetching package contents in bulk, to do local analyses. ## Test plan New E2E tests: ``` sui$ cargo nextest run -p sui-graphql-e2e-tests \ --features pg_integration \ -- packages/versioning ``` Also tested for performance against a large read replica (the query planner quotes a high estimate for the query but the actual results do not take very long to run because queries on many sub-partitions are eliminated). --- .../tests/packages/versioning.exp | 203 ++++++++++++++++-- .../tests/packages/versioning.move | 54 +++++ .../schema/current_progress_schema.graphql | 8 + crates/sui-graphql-rpc/src/types/event.rs | 12 +- .../sui-graphql-rpc/src/types/move_package.rs | 179 ++++++++++++++- crates/sui-graphql-rpc/src/types/query.rs | 31 ++- .../snapshot_tests__schema_sdl_export.snap | 8 + 7 files changed, 473 insertions(+), 22 deletions(-) diff --git a/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp b/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp index 6fa2f95e46dcf3..3803abeba10611 100644 --- a/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp +++ b/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp @@ -1,4 +1,4 @@ -processed 14 tasks +processed 15 tasks init: A: object(0,0) @@ -11,7 +11,7 @@ gas summary: computation_cost: 1000000, storage_cost: 5076800, storage_rebate: task 2 'create-checkpoint'. lines 11-11: Checkpoint created: 1 -task 3 'run-graphql'. lines 13-21: +task 3 'run-graphql'. lines 13-28: Response: { "data": { "latestPackage": { @@ -25,19 +25,43 @@ Response: { ] } } + }, + "packages": { + "nodes": [ + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000001", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000002", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000003", + "version": 1 + }, + { + "address": "0x000000000000000000000000000000000000000000000000000000000000dee9", + "version": 1 + }, + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + } + ] } } } -task 4 'upgrade'. lines 23-27: +task 4 'upgrade'. lines 30-34: created: object(4,0) mutated: object(0,0), object(1,1) gas summary: computation_cost: 1000000, storage_cost: 5251600, storage_rebate: 2595780, non_refundable_storage_fee: 26220 -task 5 'create-checkpoint'. lines 29-29: +task 5 'create-checkpoint'. lines 36-36: Checkpoint created: 2 -task 6 'run-graphql'. lines 31-39: +task 6 'run-graphql'. lines 38-53: Response: { "data": { "latestPackage": { @@ -54,19 +78,47 @@ Response: { ] } } + }, + "packages": { + "nodes": [ + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000001", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000002", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000003", + "version": 1 + }, + { + "address": "0x000000000000000000000000000000000000000000000000000000000000dee9", + "version": 1 + }, + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + }, + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + } + ] } } } -task 7 'upgrade'. lines 41-46: +task 7 'upgrade'. lines 55-60: created: object(7,0) mutated: object(0,0), object(1,1) gas summary: computation_cost: 1000000, storage_cost: 5426400, storage_rebate: 2595780, non_refundable_storage_fee: 26220 -task 8 'create-checkpoint'. lines 48-48: +task 8 'create-checkpoint'. lines 62-62: Checkpoint created: 3 -task 9 'run-graphql'. lines 50-58: +task 9 'run-graphql'. lines 64-79: Response: { "data": { "latestPackage": { @@ -86,11 +138,43 @@ Response: { ] } } + }, + "packages": { + "nodes": [ + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000001", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000002", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000003", + "version": 1 + }, + { + "address": "0x000000000000000000000000000000000000000000000000000000000000dee9", + "version": 1 + }, + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + }, + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + }, + { + "address": "0x0eae57b7a07b0548b1f6b0c309f0692828ff994e9159b541334b25582980631c", + "version": 3 + } + ] } } } -task 10 'run-graphql'. lines 60-97: +task 10 'run-graphql'. lines 81-118: Response: { "data": { "v1": { @@ -189,7 +273,7 @@ Response: { } } -task 11 'run-graphql'. lines 99-136: +task 11 'run-graphql'. lines 120-157: Response: { "data": { "v1_from_p1": { @@ -279,7 +363,7 @@ Response: { } } -task 12 'run-graphql'. lines 138-193: +task 12 'run-graphql'. lines 159-214: Response: { "data": { "v1": { @@ -417,7 +501,7 @@ Response: { } } -task 13 'run-graphql'. lines 195-223: +task 13 'run-graphql'. lines 216-244: Response: { "data": { "v0": null, @@ -428,3 +512,98 @@ Response: { "v4": null } } + +task 14 'run-graphql'. lines 246-277: +Response: { + "data": { + "before": { + "nodes": [ + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000001", + "version": 1, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 0 + } + } + } + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000002", + "version": 1, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 0 + } + } + } + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000003", + "version": 1, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 0 + } + } + } + }, + { + "address": "0x000000000000000000000000000000000000000000000000000000000000dee9", + "version": 1, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 0 + } + } + } + } + ] + }, + "after": { + "nodes": [ + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 2 + } + } + } + }, + { + "address": "0x0eae57b7a07b0548b1f6b0c309f0692828ff994e9159b541334b25582980631c", + "version": 3, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 3 + } + } + } + } + ] + }, + "between": { + "nodes": [ + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 2 + } + } + } + } + ] + } + } +} diff --git a/crates/sui-graphql-e2e-tests/tests/packages/versioning.move b/crates/sui-graphql-e2e-tests/tests/packages/versioning.move index 7b169b1ad591e9..31ba3f73d86483 100644 --- a/crates/sui-graphql-e2e-tests/tests/packages/versioning.move +++ b/crates/sui-graphql-e2e-tests/tests/packages/versioning.move @@ -18,6 +18,13 @@ module P0::m { functions { nodes { name } } } } + + packages(first: 10) { + nodes { + address + version + } + } } //# upgrade --package P0 --upgrade-capability 1,1 --sender A @@ -36,6 +43,13 @@ module P1::m { functions { nodes { name } } } } + + packages(first: 10) { + nodes { + address + version + } + } } //# upgrade --package P1 --upgrade-capability 1,1 --sender A @@ -55,6 +69,13 @@ module P2::m { functions { nodes { name } } } } + + packages(first: 10) { + nodes { + address + version + } + } } //# run-graphql @@ -221,3 +242,36 @@ module P2::m { } } } + +//# run-graphql +{ # Querying packages with checkpoint bounds + before: packages(beforeCheckpoint: 1, first: 10) { + nodes { + address + version + previousTransactionBlock { + effects { checkpoint { sequenceNumber } } + } + } + } + + after: packages(afterCheckpoint: 1, first: 10) { + nodes { + address + version + previousTransactionBlock { + effects { checkpoint { sequenceNumber } } + } + } + } + + between: packages(afterCheckpoint: 1, beforeCheckpoint: 3, first: 10) { + nodes { + address + version + previousTransactionBlock { + effects { checkpoint { sequenceNumber } } + } + } + } +} diff --git a/crates/sui-graphql-rpc/schema/current_progress_schema.graphql b/crates/sui-graphql-rpc/schema/current_progress_schema.graphql index fda68a9b8e1966..0bc651919b0b6d 100644 --- a/crates/sui-graphql-rpc/schema/current_progress_schema.graphql +++ b/crates/sui-graphql-rpc/schema/current_progress_schema.graphql @@ -2931,6 +2931,14 @@ type Query { """ objects(first: Int, after: String, last: Int, before: String, filter: ObjectFilter): ObjectConnection! """ + The Move packages that exist in the network, optionally filtered to be strictly before + `beforeCheckpoint` and/or strictly after `afterCheckpoint`. + + This query will return all versions of a given user package that appear between the + specified checkpoints, but only records the latest versions of system packages. + """ + packages(first: Int, after: String, last: Int, before: String, afterCheckpoint: Int, beforeCheckpoint: Int): MovePackageConnection! + """ Fetch the protocol config by protocol version (defaults to the latest protocol version known to the GraphQL service). """ diff --git a/crates/sui-graphql-rpc/src/types/event.rs b/crates/sui-graphql-rpc/src/types/event.rs index 16284f01618cd2..6b7ba8ee8b3c26 100644 --- a/crates/sui-graphql-rpc/src/types/event.rs +++ b/crates/sui-graphql-rpc/src/types/event.rs @@ -143,13 +143,13 @@ impl Event { /// checkpoint sequence numbers as the cursor to determine the correct page of results. The /// query can optionally be further `filter`-ed by the `EventFilter`. /// - /// The `checkpoint_viewed_at` parameter is represents the checkpoint sequence number at which - /// this page was queried for. Each entity returned in the connection will inherit this - /// checkpoint, so that when viewing that entity's state, it will be from the reference of this - /// checkpoint_viewed_at parameter. + /// The `checkpoint_viewed_at` parameter represents the checkpoint sequence number at which this + /// page was queried. Each entity returned in the connection will inherit this checkpoint, so + /// that when viewing that entity's state, it will be as if it is being viewed at this + /// checkpoint. /// - /// If the `Page` is set, then this function will defer to the `checkpoint_viewed_at` in - /// the cursor if they are consistent. + /// The cursors in `page` may also include checkpoint viewed at fields. If these are set, they + /// take precedence over the checkpoint that pagination is being conducted in. pub(crate) async fn paginate( db: &Db, page: Page, diff --git a/crates/sui-graphql-rpc/src/types/move_package.rs b/crates/sui-graphql-rpc/src/types/move_package.rs index a8bfee31452fb4..cdad8b11e217a7 100644 --- a/crates/sui-graphql-rpc/src/types/move_package.rs +++ b/crates/sui-graphql-rpc/src/types/move_package.rs @@ -7,7 +7,7 @@ use super::balance::{self, Balance}; use super::base64::Base64; use super::big_int::BigInt; use super::coin::Coin; -use super::cursor::{JsonCursor, Page}; +use super::cursor::{BcsCursor, JsonCursor, Page, RawPaginated, Target}; use super::move_module::MoveModule; use super::move_object::MoveObject; use super::object::{self, Object, ObjectFilter, ObjectImpl, ObjectOwner, ObjectStatus}; @@ -17,14 +17,19 @@ use super::sui_address::SuiAddress; use super::suins_registration::{DomainFormat, SuinsRegistration}; use super::transaction_block::{self, TransactionBlock, TransactionBlockFilter}; use super::type_filter::ExactTypeFilter; -use crate::consistency::ConsistentNamedCursor; +use crate::consistency::{Checkpointed, ConsistentNamedCursor}; use crate::data::{DataLoader, Db, DbConnection, QueryExecutor}; use crate::error::Error; +use crate::raw_query::RawQuery; use crate::types::sui_address::addr; +use crate::{filter, query}; use async_graphql::connection::{Connection, CursorType, Edge}; use async_graphql::dataloader::Loader; use async_graphql::*; -use diesel::{BoolExpressionMethods, ExpressionMethods, JoinOnDsl, QueryDsl}; +use diesel::prelude::QueryableByName; +use diesel::{BoolExpressionMethods, ExpressionMethods, JoinOnDsl, QueryDsl, Selectable}; +use serde::{Deserialize, Serialize}; +use sui_indexer::models::objects::StoredHistoryObject; use sui_indexer::schema::packages; use sui_package_resolver::{error::Error as PackageCacheError, Package as ParsedMovePackage}; use sui_types::is_system_package; @@ -86,9 +91,31 @@ struct TypeOrigin { defining_id: SuiAddress, } +/// A wrapper around the stored representation of a package, used to implement pagination-related +/// traits. +#[derive(Selectable, QueryableByName)] +#[diesel(table_name = packages)] +struct StoredHistoryPackage { + original_id: Vec, + #[diesel(embed)] + object: StoredHistoryObject, +} + pub(crate) struct MovePackageDowncastError; pub(crate) type CModule = JsonCursor; +pub(crate) type Cursor = BcsCursor; + +/// The inner struct for the `MovePackage` cursor. The package is identified by the checkpoint it +/// was created in, its original ID, and its version, and the `checkpoint_viewed_at` specifies the +/// checkpoint snapshot that the data came from. +#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] +pub(crate) struct PackageCursor { + pub checkpoint_sequence_number: u64, + pub original_id: Vec, + pub package_version: u64, + pub checkpoint_viewed_at: u64, +} /// DataLoader key for fetching the storage ID of the (user) package that shares an original (aka /// runtime) ID with the package stored at `package_id`, and whose version is `version`. @@ -569,6 +596,152 @@ impl MovePackage { Error::Internal(format!("{address} is not a package")) })?)) } + + /// Query the database for a `page` of Move packages. The Page uses the checkpoint sequence + /// number the package was created at, its original ID, and its version as the cursor. The query + /// can optionally be filtered by a bound on the checkpoints the packages were created in. + /// + /// The `checkpoint_viewed_at` parameter represents the checkpoint sequence number at which this + /// page was queried. Each entity returned in the connection will inherit this checkpoint, so + /// that when viewing that entity's state, it will be as if it is being viewed at this + /// checkpoint. + /// + /// The cursors in `page` may also include checkpoint viewed at fields. If these are set, they + /// take precedence over the checkpoint that pagination is being conducted in. + pub(crate) async fn paginate_by_checkpoint( + db: &Db, + page: Page, + after_checkpoint: Option, + before_checkpoint: Option, + checkpoint_viewed_at: u64, + ) -> Result, Error> { + let cursor_viewed_at = page.validate_cursor_consistency()?; + let checkpoint_viewed_at = cursor_viewed_at.unwrap_or(checkpoint_viewed_at); + + // Clamp the "before checkpoint" bound by "checkpoint viewed at". + let before_checkpoint = + (checkpoint_viewed_at + 1).min(before_checkpoint.unwrap_or(u64::MAX)); + + let (prev, next, results) = db + .execute(move |conn| { + let mut q = query!( + r#" + SELECT + p.original_id, + o.* + FROM + packages p + INNER JOIN + objects_history o + ON + p.package_id = o.object_id + AND p.package_version = o.object_version + AND p.checkpoint_sequence_number = o.checkpoint_sequence_number + "# + ); + + q = filter!( + q, + format!("o.checkpoint_sequence_number < {before_checkpoint}") + ); + if let Some(after) = after_checkpoint { + q = filter!(q, format!("{after} < o.checkpoint_sequence_number")); + } + + page.paginate_raw_query::(conn, checkpoint_viewed_at, q) + }) + .await?; + + let mut conn = Connection::new(prev, next); + + // The "checkpoint viewed at" sets a consistent upper bound for the nested queries. + for stored in results { + let cursor = stored.cursor(checkpoint_viewed_at).encode_cursor(); + let package = + MovePackage::try_from_stored_history_object(stored.object, checkpoint_viewed_at)?; + conn.edges.push(Edge::new(cursor, package)); + } + + Ok(conn) + } + + /// `checkpoint_viewed_at` points to the checkpoint snapshot that this `MovePackage` came from. + /// This is stored in the `MovePackage` so that related fields from the package are read from + /// the same checkpoint (consistently). + pub(crate) fn try_from_stored_history_object( + history_object: StoredHistoryObject, + checkpoint_viewed_at: u64, + ) -> Result { + let object = Object::try_from_stored_history_object(history_object, checkpoint_viewed_at)?; + Self::try_from(&object).map_err(|_| Error::Internal("Not a package!".to_string())) + } +} + +impl Checkpointed for Cursor { + fn checkpoint_viewed_at(&self) -> u64 { + self.checkpoint_viewed_at + } +} + +impl RawPaginated for StoredHistoryPackage { + fn filter_ge(cursor: &Cursor, query: RawQuery) -> RawQuery { + filter!( + query, + format!( + "o.checkpoint_sequence_number > {cp} OR (\ + o.checkpoint_sequence_number = {cp} AND + p.original_id > '\\x{id}'::bytea OR (\ + p.original_id = '\\x{id}'::bytea AND \ + p.package_version >= {pv}\ + ))", + cp = cursor.checkpoint_sequence_number, + id = hex::encode(&cursor.original_id), + pv = cursor.package_version, + ) + ) + } + + fn filter_le(cursor: &Cursor, query: RawQuery) -> RawQuery { + filter!( + query, + format!( + "o.checkpoint_sequence_number < {cp} OR (\ + o.checkpoint_sequence_number = {cp} AND + p.original_id < '\\x{id}'::bytea OR (\ + p.original_id = '\\x{id}'::bytea AND \ + p.package_version <= {pv}\ + ))", + cp = cursor.checkpoint_sequence_number, + id = hex::encode(&cursor.original_id), + pv = cursor.package_version, + ) + ) + } + + fn order(asc: bool, query: RawQuery) -> RawQuery { + if asc { + query + .order_by("o.checkpoint_sequence_number ASC") + .order_by("p.original_id ASC") + .order_by("p.package_version ASC") + } else { + query + .order_by("o.checkpoint_sequence_number DESC") + .order_by("p.original_id DESC") + .order_by("p.package_version DESC") + } + } +} + +impl Target for StoredHistoryPackage { + fn cursor(&self, checkpoint_viewed_at: u64) -> Cursor { + Cursor::new(PackageCursor { + checkpoint_sequence_number: self.object.checkpoint_sequence_number as u64, + original_id: self.original_id.clone(), + package_version: self.object.object_version as u64, + checkpoint_viewed_at, + }) + } } #[async_trait::async_trait] diff --git a/crates/sui-graphql-rpc/src/types/query.rs b/crates/sui-graphql-rpc/src/types/query.rs index 431c3bd7ed8f84..39ed2eb7897435 100644 --- a/crates/sui-graphql-rpc/src/types/query.rs +++ b/crates/sui-graphql-rpc/src/types/query.rs @@ -12,7 +12,7 @@ use sui_sdk::SuiClient; use sui_types::transaction::{TransactionData, TransactionKind}; use sui_types::{gas_coin::GAS, transaction::TransactionDataAPI, TypeTag}; -use super::move_package::MovePackage; +use super::move_package::{self, MovePackage}; use super::suins_registration::NameService; use super::{ address::Address, @@ -411,6 +411,35 @@ impl Query { .extend() } + /// The Move packages that exist in the network, optionally filtered to be strictly before + /// `beforeCheckpoint` and/or strictly after `afterCheckpoint`. + /// + /// This query will return all versions of a given user package that appear between the + /// specified checkpoints, but only records the latest versions of system packages. + async fn packages( + &self, + ctx: &Context<'_>, + first: Option, + after: Option, + last: Option, + before: Option, + after_checkpoint: Option, + before_checkpoint: Option, + ) -> Result> { + let Watermark { checkpoint, .. } = *ctx.data()?; + + let page = Page::from_params(ctx.data_unchecked(), first, after, last, before)?; + MovePackage::paginate_by_checkpoint( + ctx.data_unchecked(), + page, + after_checkpoint, + before_checkpoint, + checkpoint, + ) + .await + .extend() + } + /// Fetch the protocol config by protocol version (defaults to the latest protocol /// version known to the GraphQL service). async fn protocol_config( diff --git a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap index f0d6a06b1735a2..5a20d112252362 100644 --- a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap +++ b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap @@ -2935,6 +2935,14 @@ type Query { """ objects(first: Int, after: String, last: Int, before: String, filter: ObjectFilter): ObjectConnection! """ + The Move packages that exist in the network, optionally filtered to be strictly before + `beforeCheckpoint` and/or strictly after `afterCheckpoint`. + + This query will return all versions of a given user package that appear between the + specified checkpoints, but only records the latest versions of system packages. + """ + packages(first: Int, after: String, last: Int, before: String, afterCheckpoint: Int, beforeCheckpoint: Int): MovePackageConnection! + """ Fetch the protocol config by protocol version (defaults to the latest protocol version known to the GraphQL service). """