diff --git a/CHANGELOG.md b/CHANGELOG.md index 059ff79d35..02dfd0a4d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,33 @@ Update this for each program release and mainnet deployment. ## not on mainnet -### v0.23.0, 2024-3- +### v0.24.0, 2024-4- + +- Allow skipping banks and invalid oracles when computing health (#891) + + This is only possible when we know for sure that the operation would not put the account into negative health zone. + +- Add support for Raydium CLMM as oracle fallback (#856) + +- Add a `TokenBalanceLog` when charging collateral fees (#894) + +- Withdraw instruction: remove overflow error and return appropriate error message instead (#910) + +- Banks: add more safety checks (#895) + +- Add a health check instruction (#913) + + Assert in a transaction that operation run on a mango account does not reduce it's health below a specified amount. + +- Add a sequence check instruction (#909) + + Assert that a transaction was emitted and run with a correct view of the current mango state. + +## mainnet + +### v0.23.0, 2024-3-8 + +Deployment: Mar 8, 2024 at 12:10:52 Central European Standard Time, https://explorer.solana.com/tx/6MXGookZoYGMYb7tWrrmgZzVA13HJimHNqwHRVFeqL9YpQD7YasH1pQn4MSQTK1o13ixKTGFxwZsviUzmHzzP9m - Allow disabling asset liquidations for tokens (#867) @@ -26,8 +52,6 @@ Update this for each program release and mainnet deployment. - Flash loan: Add a "swap without flash loan fees" option (#882) - Cleanup, tests and minor (#878, #875, #854, #838, #895) -## mainnet - ### v0.22.0, 2024-3-3 Deployment: Mar 3, 2024 at 23:52:08 Central European Standard Time, https://explorer.solana.com/tx/3MpEMU12Pv7RpSnwfShoM9sbyr41KAEeJFCVx9ypkq8nuK8Q5vm7CRLkdhH3u91yQ4k44a32armZHaoYguX6NqsY diff --git a/Cargo.lock b/Cargo.lock index a54abdb97f..7b423022eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3462,6 +3462,7 @@ dependencies = [ "atty", "base64 0.13.1", "bincode", + "borsh 0.10.3", "clap 3.2.25", "derive_builder", "fixed 1.11.0 (git+https://github.com/blockworks-foundation/fixed.git?branch=v1.11.0-borsh0_10-mango)", @@ -3529,6 +3530,7 @@ dependencies = [ "async-channel", "async-stream 0.2.1", "async-trait", + "borsh 0.10.3", "bs58 0.3.1", "bytemuck", "bytes 1.5.0", @@ -3540,6 +3542,7 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", + "indexmap 2.0.0", "itertools", "jemallocator", "jsonrpc-core 18.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3556,6 +3559,7 @@ dependencies = [ "serum_dex 0.5.10 (git+https://github.com/openbook-dex/program.git)", "shellexpand", "solana-account-decoder", + "solana-address-lookup-table-program", "solana-client", "solana-logger", "solana-rpc", diff --git a/audits/Audit_OtterSec_Mango_v0.24.0.pdf b/audits/Audit_OtterSec_Mango_v0.24.0.pdf new file mode 100644 index 0000000000..ed32f60181 Binary files /dev/null and b/audits/Audit_OtterSec_Mango_v0.24.0.pdf differ diff --git a/bin/cli/src/main.rs b/bin/cli/src/main.rs index 06ab893be4..fe3f5562c0 100644 --- a/bin/cli/src/main.rs +++ b/bin/cli/src/main.rs @@ -92,6 +92,31 @@ struct JupiterSwap { rpc: Rpc, } +#[derive(Args, Debug, Clone)] +struct SanctumSwap { + #[clap(long)] + account: String, + + /// also pays for everything + #[clap(short, long)] + owner: String, + + #[clap(long)] + input_mint: String, + + #[clap(long)] + output_mint: String, + + #[clap(short, long)] + amount: u64, + + #[clap(short, long, default_value = "50")] + max_slippage_bps: u64, + + #[clap(flatten)] + rpc: Rpc, +} + #[derive(ArgEnum, Clone, Debug)] #[repr(u8)] pub enum CliSide { @@ -189,6 +214,7 @@ enum Command { CreateAccount(CreateAccount), Deposit(Deposit), JupiterSwap(JupiterSwap), + SanctumSwap(SanctumSwap), GroupAddress { #[clap(short, long)] creator: String, @@ -312,6 +338,19 @@ async fn main() -> Result<(), anyhow::Error> { .await?; println!("{}", txsig); } + Command::SanctumSwap(cmd) => { + let client = cmd.rpc.client(Some(&cmd.owner))?; + let account = pubkey_from_cli(&cmd.account); + let owner = Arc::new(keypair_from_cli(&cmd.owner)); + let input_mint = pubkey_from_cli(&cmd.input_mint); + let output_mint = pubkey_from_cli(&cmd.output_mint); + let client = MangoClient::new_for_existing_account(client, account, owner).await?; + let txsig = client + .sanctum() + .swap(input_mint, output_mint, cmd.max_slippage_bps, cmd.amount) + .await?; + println!("{}", txsig); + } Command::GroupAddress { creator, num } => { let creator = pubkey_from_cli(&creator); println!("{}", MangoClient::group_for_admin(creator, num)); diff --git a/bin/cli/src/save_snapshot.rs b/bin/cli/src/save_snapshot.rs index 575b900bbc..edd6e63e59 100644 --- a/bin/cli/src/save_snapshot.rs +++ b/bin/cli/src/save_snapshot.rs @@ -67,7 +67,7 @@ pub async fn save_snapshot( .await?; // Getting solana account snapshots via jsonrpc - snapshot_source::start( + let snapshot_job = snapshot_source::start( snapshot_source::Config { rpc_http_url: rpc_url.clone(), mango_group, @@ -79,6 +79,11 @@ pub async fn save_snapshot( extra_accounts, account_update_sender, ); + tokio::spawn(async move { + let res = snapshot_job.await; + tracing::error!("Snapshot job exited, terminating process.. ({:?})", res); + std::process::exit(-1); + }); let mut chain_data = chain_data::ChainData::new(); diff --git a/bin/liquidator/Cargo.toml b/bin/liquidator/Cargo.toml index d591bd37b2..9e13f12ef4 100644 --- a/bin/liquidator/Cargo.toml +++ b/bin/liquidator/Cargo.toml @@ -42,6 +42,7 @@ shellexpand = "2.1.0" solana-account-decoder = { workspace = true } solana-client = { workspace = true } solana-logger = { workspace = true } +solana-address-lookup-table-program = "~1.16.7" solana-rpc = { workspace = true } solana-sdk = { workspace = true } tokio = { version = "1", features = ["full"] } @@ -49,4 +50,6 @@ tokio-stream = { version = "0.1.9"} tokio-tungstenite = "0.16.1" tracing = "0.1" regex = "1.9.5" -hdrhistogram = "7.5.4" \ No newline at end of file +hdrhistogram = "7.5.4" +indexmap = "2.0.0" +borsh = { version = "0.10.3", features = ["const-generics"] } diff --git a/bin/liquidator/src/cli_args.rs b/bin/liquidator/src/cli_args.rs index 53ea01fad8..c6c6b0a282 100644 --- a/bin/liquidator/src/cli_args.rs +++ b/bin/liquidator/src/cli_args.rs @@ -1,7 +1,7 @@ use crate::trigger_tcs; use anchor_lang::prelude::Pubkey; use clap::Parser; -use mango_v4_client::{jupiter, priority_fees_cli}; +use mango_v4_client::{priority_fees_cli, swap}; use std::collections::HashSet; #[derive(Parser, Debug)] @@ -28,11 +28,11 @@ pub(crate) enum JupiterVersionArg { V6, } -impl From for jupiter::Version { +impl From for swap::Version { fn from(a: JupiterVersionArg) -> Self { match a { - JupiterVersionArg::Mock => jupiter::Version::Mock, - JupiterVersionArg::V6 => jupiter::Version::V6, + JupiterVersionArg::Mock => swap::Version::Mock, + JupiterVersionArg::V6 => swap::Version::V6, } } } @@ -121,6 +121,12 @@ pub struct Cli { #[clap(long, env, value_parser, value_delimiter = ',')] pub(crate) rebalance_alternate_jupiter_route_tokens: Option>, + /// query sanctum for routes to and from these tokens + /// + /// These routes will only be used when trying to rebalance a LST token + #[clap(long, env, value_parser, value_delimiter = ',')] + pub(crate) rebalance_alternate_sanctum_route_tokens: Option>, + /// When closing borrows, the rebalancer can't close token positions exactly. /// Instead it purchases too much and then gets rid of the excess in a second step. /// If this is 0.05, then it'll swap borrow_value * (1 + 0.05) quote token into borrow token. @@ -130,12 +136,25 @@ pub struct Cli { #[clap(long, env, default_value = "30")] pub(crate) rebalance_refresh_timeout_secs: u64, + #[clap(long, env, value_enum, default_value = "false")] + pub(crate) rebalance_using_limit_order: BoolArg, + + /// distance (in bps) from oracle price at which to place order for rebalancing + #[clap(long, env, default_value = "100")] + pub(crate) rebalance_limit_order_distance_from_oracle_price_bps: u64, + /// if taking tcs orders is enabled /// /// typically only disabled for tests where swaps are unavailable #[clap(long, env, value_enum, default_value = "true")] pub(crate) take_tcs: BoolArg, + #[clap(long, env, default_value = "30")] + pub(crate) tcs_refresh_timeout_secs: u64, + + #[clap(long, env, default_value = "1000")] + pub(crate) tcs_check_interval_ms: u64, + /// profit margin at which to take tcs orders #[clap(long, env, default_value = "0.0005")] pub(crate) tcs_profit_fraction: f64, @@ -178,6 +197,10 @@ pub struct Cli { #[clap(long, env, default_value = "https://quote-api.jup.ag/v6")] pub(crate) jupiter_v6_url: String, + /// override the jupiter http request timeout + #[clap(long, env, default_value = "30")] + pub(crate) jupiter_timeout_secs: u64, + /// provide a jupiter token, currently only for jup v6 #[clap(long, env, default_value = "")] pub(crate) jupiter_token: String, @@ -191,6 +214,12 @@ pub struct Cli { #[clap(long, env, value_enum, default_value = "true")] pub(crate) telemetry: BoolArg, + /// if liquidation is enabled + /// + /// might be used to run an instance of liquidator dedicated to TCS and another one for liquidation + #[clap(long, env, value_enum, default_value = "true")] + pub(crate) liquidation_enabled: BoolArg, + /// liquidation refresh timeout in secs #[clap(long, env, default_value = "30")] pub(crate) liquidation_refresh_timeout_secs: u8, @@ -216,4 +245,20 @@ pub struct Cli { /// how long should it wait before logging an oracle error again (for the same token) #[clap(long, env, default_value = "30")] pub(crate) skip_oracle_error_in_logs_duration_secs: u64, + + /// max number of liquidation/tcs to do concurrently + #[clap(long, env, default_value = "5")] + pub(crate) max_parallel_operations: u64, + + /// Also use sanctum for rebalancing + #[clap(long, env, value_enum, default_value = "false")] + pub(crate) sanctum_enabled: BoolArg, + + /// override the url to sanctum + #[clap(long, env, default_value = "https://api.sanctum.so/v1")] + pub(crate) sanctum_url: String, + + /// override the sanctum http request timeout + #[clap(long, env, default_value = "30")] + pub(crate) sanctum_timeout_secs: u64, } diff --git a/bin/liquidator/src/liquidate.rs b/bin/liquidator/src/liquidate.rs index c355aaa19f..06c3d1f018 100644 --- a/bin/liquidator/src/liquidate.rs +++ b/bin/liquidator/src/liquidate.rs @@ -9,6 +9,7 @@ use mango_v4_client::{chain_data, MangoClient, PreparedInstructions}; use solana_sdk::signature::Signature; use futures::{stream, StreamExt, TryStreamExt}; +use mango_v4::accounts_ix::HealthCheckKind::MaintRatio; use rand::seq::SliceRandom; use tracing::*; use {anyhow::Context, fixed::types::I80F48, solana_sdk::pubkey::Pubkey}; @@ -260,7 +261,22 @@ impl<'a> LiquidateHelper<'a> { ) .await .context("creating perp_liq_base_or_positive_pnl_instruction")?; + liq_ixs.cu = liq_ixs.cu.max(self.config.compute_limit_for_liq_ix); + + let liqor = &self.client.mango_account().await?; + liq_ixs.append( + self.client + .health_check_instruction( + liqor, + self.config.min_health_ratio, + vec![], + vec![*perp_market_index], + MaintRatio, + ) + .await?, + ); + let txsig = self .client .send_and_confirm_owner_tx(liq_ixs.to_instructions()) @@ -501,6 +517,20 @@ impl<'a> LiquidateHelper<'a> { .await .context("creating liq_token_with_token ix")?; liq_ixs.cu = liq_ixs.cu.max(self.config.compute_limit_for_liq_ix); + + let liqor = self.client.mango_account().await?; + liq_ixs.append( + self.client + .health_check_instruction( + &liqor, + self.config.min_health_ratio, + vec![asset_token_index, liab_token_index], + vec![], + MaintRatio, + ) + .await?, + ); + let txsig = self .client .send_and_confirm_owner_tx(liq_ixs.to_instructions()) @@ -651,14 +681,11 @@ impl<'a> LiquidateHelper<'a> { } #[allow(clippy::too_many_arguments)] -pub async fn maybe_liquidate_account( +pub async fn can_liquidate_account( mango_client: &MangoClient, account_fetcher: &chain_data::AccountFetcher, pubkey: &Pubkey, - config: &Config, ) -> anyhow::Result { - let liqor_min_health_ratio = I80F48::from_num(config.min_health_ratio); - let account = account_fetcher.fetch_mango_account(pubkey)?; let health_cache = mango_client .health_cache(&account) @@ -675,6 +702,18 @@ pub async fn maybe_liquidate_account( "possible candidate", ); + Ok(true) +} + +#[allow(clippy::too_many_arguments)] +pub async fn maybe_liquidate_account( + mango_client: &MangoClient, + account_fetcher: &chain_data::AccountFetcher, + pubkey: &Pubkey, + config: &Config, +) -> anyhow::Result { + let liqor_min_health_ratio = I80F48::from_num(config.min_health_ratio); + // Fetch a fresh account and re-compute // This is -- unfortunately -- needed because the websocket streams seem to not // be great at providing timely updates to the account data. diff --git a/bin/liquidator/src/liquidation_state.rs b/bin/liquidator/src/liquidation_state.rs new file mode 100644 index 0000000000..aedae78908 --- /dev/null +++ b/bin/liquidator/src/liquidation_state.rs @@ -0,0 +1,238 @@ +use crate::cli_args::Cli; +use crate::metrics::Metrics; +use crate::unwrappable_oracle_error::UnwrappableOracleError; +use crate::{liquidate, LiqErrorType, SharedState}; +use anchor_lang::prelude::Pubkey; +use itertools::Itertools; +use mango_v4::state::TokenIndex; +use mango_v4_client::error_tracking::ErrorTracking; +use mango_v4_client::{chain_data, MangoClient, MangoClientError}; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; +use tokio::task::JoinHandle; +use tracing::{error, trace, warn}; + +#[derive(Clone)] +pub struct LiquidationState { + pub mango_client: Arc, + pub account_fetcher: Arc, + pub liquidation_config: liquidate::Config, + + pub errors: Arc>>, + pub oracle_errors: Arc>>, +} + +impl LiquidationState { + async fn find_candidates( + &mut self, + accounts_iter: impl Iterator, + action: impl Fn(Pubkey) -> anyhow::Result<()>, + ) -> anyhow::Result { + let mut found_counter = 0u64; + use rand::seq::SliceRandom; + + let mut accounts = accounts_iter.collect::>(); + { + let mut rng = rand::thread_rng(); + accounts.shuffle(&mut rng); + } + + for pubkey in accounts { + if self.should_skip_execution(pubkey) { + continue; + } + + let result = + liquidate::can_liquidate_account(&self.mango_client, &self.account_fetcher, pubkey) + .await; + + self.log_or_ignore_error(&result, pubkey); + + if result.unwrap_or(false) { + action(*pubkey)?; + found_counter = found_counter + 1; + } + } + + Ok(found_counter) + } + + fn should_skip_execution(&mut self, pubkey: &Pubkey) -> bool { + let now = Instant::now(); + let error_tracking = &mut self.errors; + + // Skip a pubkey if there've been too many errors recently + if let Some(error_entry) = + error_tracking + .read() + .unwrap() + .had_too_many_errors(LiqErrorType::Liq, pubkey, now) + { + trace!( + %pubkey, + error_entry.count, + "skip checking account for liquidation, had errors recently", + ); + return true; + } + + false + } + + fn log_or_ignore_error(&mut self, result: &anyhow::Result, pubkey: &Pubkey) { + let error_tracking = &mut self.errors; + + if let Err(err) = result.as_ref() { + if let Some((ti, ti_name)) = err.try_unwrap_oracle_error() { + if self + .oracle_errors + .read() + .unwrap() + .had_too_many_errors(LiqErrorType::Liq, &ti, Instant::now()) + .is_none() + { + warn!( + "{:?} recording oracle error for token {} {}", + chrono::offset::Utc::now(), + ti_name, + ti + ); + } + + self.oracle_errors + .write() + .unwrap() + .record(LiqErrorType::Liq, &ti, err.to_string()); + return; + } + + // Keep track of pubkeys that had errors + error_tracking + .write() + .unwrap() + .record(LiqErrorType::Liq, pubkey, err.to_string()); + + // Not all errors need to be raised to the user's attention. + let mut is_error = true; + + // Simulation errors due to liqee precondition failures on the liquidation instructions + // will commonly happen if our liquidator is late or if there are chain forks. + match err.downcast_ref::() { + Some(MangoClientError::SendTransactionPreflightFailure { logs, .. }) => { + if logs.iter().any(|line| { + line.contains("HealthMustBeNegative") || line.contains("IsNotBankrupt") + }) { + is_error = false; + } + } + _ => {} + }; + if is_error { + error!("liquidating account {}: {:?}", pubkey, err); + } else { + trace!("liquidating account {}: {:?}", pubkey, err); + } + } else { + error_tracking + .write() + .unwrap() + .clear(LiqErrorType::Liq, pubkey); + } + } + + pub async fn maybe_liquidate_and_log_error(&mut self, pubkey: &Pubkey) -> anyhow::Result { + if self.should_skip_execution(pubkey) { + return Ok(false); + } + + let result = liquidate::maybe_liquidate_account( + &self.mango_client, + &self.account_fetcher, + pubkey, + &self.liquidation_config, + ) + .await; + + self.log_or_ignore_error(&result, pubkey); + return result; + } +} + +pub fn spawn_liquidation_job( + cli: &Cli, + shared_state: &Arc>, + tx_trigger_sender: async_channel::Sender<()>, + mut liquidation: Box, + metrics: &Metrics, +) -> JoinHandle<()> { + tokio::spawn({ + let mut interval = + mango_v4_client::delay_interval(Duration::from_millis(cli.check_interval_ms)); + let mut metric_liquidation_check = metrics.register_latency("liquidation_check".into()); + let mut metric_liquidation_start_end = + metrics.register_latency("liquidation_start_end".into()); + + let mut liquidation_start_time = None; + + let shared_state = shared_state.clone(); + async move { + loop { + interval.tick().await; + + let account_addresses = { + let mut state = shared_state.write().unwrap(); + if !state.one_snapshot_done { + // discard first latency info as it will skew data too much + state.oldest_chain_event_reception_time = None; + continue; + } + if state.oldest_chain_event_reception_time.is_none() + && liquidation_start_time.is_none() + { + // no new update, skip computing + continue; + } + + state.mango_accounts.iter().cloned().collect_vec() + }; + + liquidation.errors.write().unwrap().update(); + liquidation.oracle_errors.write().unwrap().update(); + + if liquidation_start_time.is_none() { + liquidation_start_time = Some(Instant::now()); + } + + let found_candidates = liquidation + .find_candidates(account_addresses.iter(), |p| { + if shared_state + .write() + .unwrap() + .liquidation_candidates_accounts + .insert(p) + { + tx_trigger_sender.try_send(())?; + } + + Ok(()) + }) + .await + .unwrap(); + + if found_candidates > 0 { + tracing::debug!("found {} candidates for liquidation", found_candidates); + } + + let mut state = shared_state.write().unwrap(); + let reception_time = state.oldest_chain_event_reception_time.unwrap(); + let current_time = Instant::now(); + + state.oldest_chain_event_reception_time = None; + + metric_liquidation_check.push(current_time - reception_time); + metric_liquidation_start_end.push(current_time - liquidation_start_time.unwrap()); + liquidation_start_time = None; + } + } + }) +} diff --git a/bin/liquidator/src/main.rs b/bin/liquidator/src/main.rs index 758736936b..846d8c9b2f 100644 --- a/bin/liquidator/src/main.rs +++ b/bin/liquidator/src/main.rs @@ -4,33 +4,40 @@ use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; use anchor_client::Cluster; -use anyhow::Context; use clap::Parser; +use futures_util::StreamExt; use mango_v4::state::{PerpMarketIndex, TokenIndex}; -use mango_v4_client::AsyncChannelSendUnlessFull; use mango_v4_client::{ account_update_stream, chain_data, error_tracking::ErrorTracking, keypair_from_cli, - snapshot_source, websocket_source, Client, MangoClient, MangoClientError, MangoGroupContext, + snapshot_source, websocket_source, Client, MangoClient, MangoGroupContext, TransactionBuilderConfig, }; +use crate::cli_args::{BoolArg, Cli, CliDotenv}; +use crate::liquidation_state::LiquidationState; +use crate::rebalance::Rebalancer; +use crate::tcs_state::TcsState; +use crate::token_swap_info::TokenSwapInfoUpdater; use itertools::Itertools; use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::pubkey::Pubkey; use solana_sdk::signer::Signer; +use tokio::task::JoinHandle; use tracing::*; pub mod cli_args; pub mod liquidate; +mod liquidation_state; pub mod metrics; pub mod rebalance; +mod tcs_state; pub mod telemetry; pub mod token_swap_info; pub mod trigger_tcs; +mod tx_sender; mod unwrappable_oracle_error; pub mod util; -use crate::unwrappable_oracle_error::UnwrappableOracleError; use crate::util::{is_mango_account, is_mint_info, is_perp_market}; // jemalloc seems to be better at keeping the memory footprint reasonable over @@ -69,7 +76,7 @@ async fn main() -> anyhow::Result<()> { // Client setup // let liqor_owner = Arc::new(keypair_from_cli(&cli.liqor_owner)); - let rpc_url = cli.rpc_url; + let rpc_url = cli.rpc_url.clone(); let ws_url = rpc_url.replace("https", "wss"); let rpc_timeout = Duration::from_secs(10); let cluster = Cluster::Custom(rpc_url.clone(), ws_url.clone()); @@ -79,8 +86,11 @@ async fn main() -> anyhow::Result<()> { .commitment(commitment) .fee_payer(Some(liqor_owner.clone())) .timeout(rpc_timeout) - .jupiter_v6_url(cli.jupiter_v6_url) - .jupiter_token(cli.jupiter_token) + .jupiter_timeout(Duration::from_secs(cli.jupiter_timeout_secs)) + .jupiter_v6_url(cli.jupiter_v6_url.clone()) + .jupiter_token(cli.jupiter_token.clone()) + .sanctum_url(cli.sanctum_url.clone()) + .sanctum_timeout(Duration::from_secs(cli.sanctum_timeout_secs)) .transaction_builder_config( TransactionBuilderConfig::builder() .priority_fee_provider(prio_provider) @@ -89,7 +99,7 @@ async fn main() -> anyhow::Result<()> { .build() .unwrap(), ) - .override_send_transaction_urls(cli.override_send_transaction_url) + .override_send_transaction_urls(cli.override_send_transaction_url.clone()) .build() .unwrap(); @@ -161,7 +171,7 @@ async fn main() -> anyhow::Result<()> { // Getting solana account snapshots via jsonrpc // FUTURE: of what to fetch a snapshot - should probably take as an input - snapshot_source::start( + let snapshot_job = snapshot_source::start( snapshot_source::Config { rpc_http_url: rpc_url.clone(), mango_group, @@ -207,17 +217,18 @@ async fn main() -> anyhow::Result<()> { compute_limit_for_liq_ix: cli.compute_limit_for_liquidation, max_cu_per_transaction: 1_000_000, refresh_timeout: Duration::from_secs(cli.liquidation_refresh_timeout_secs as u64), - only_allowed_tokens: cli_args::cli_to_hashset::(cli.only_allow_tokens), - forbidden_tokens: cli_args::cli_to_hashset::(cli.forbidden_tokens), + only_allowed_tokens: cli_args::cli_to_hashset::(cli.only_allow_tokens.clone()), + forbidden_tokens: cli_args::cli_to_hashset::(cli.forbidden_tokens.clone()), only_allowed_perp_markets: cli_args::cli_to_hashset::( - cli.liquidation_only_allow_perp_markets, + cli.liquidation_only_allow_perp_markets.clone(), ), forbidden_perp_markets: cli_args::cli_to_hashset::( - cli.liquidation_forbidden_perp_markets, + cli.liquidation_forbidden_perp_markets.clone(), ), }; let tcs_config = trigger_tcs::Config { + refresh_timeout: Duration::from_secs(cli.tcs_refresh_timeout_secs), min_health_ratio: cli.min_health_ratio, max_trigger_quote_amount: (cli.tcs_max_trigger_amount * 1e6) as u64, compute_limit_for_trigger: cli.compute_limit_for_tcs, @@ -234,46 +245,82 @@ async fn main() -> anyhow::Result<()> { forbidden_tokens: liq_config.forbidden_tokens.clone(), }; - let mut rebalance_interval = tokio::time::interval(Duration::from_secs(30)); let (rebalance_trigger_sender, rebalance_trigger_receiver) = async_channel::bounded::<()>(1); + let (tx_tcs_trigger_sender, tx_tcs_trigger_receiver) = async_channel::unbounded::<()>(); + let (tx_liq_trigger_sender, tx_liq_trigger_receiver) = async_channel::unbounded::<()>(); + + if cli.rebalance_using_limit_order == BoolArg::True && !signer_is_owner { + warn!("Can't withdraw dust to liqor account if delegate and using limit orders for rebalancing"); + } + let rebalance_config = rebalance::Config { enabled: cli.rebalance == BoolArg::True, slippage_bps: cli.rebalance_slippage_bps, borrow_settle_excess: (1f64 + cli.rebalance_borrow_settle_excess).max(1f64), refresh_timeout: Duration::from_secs(cli.rebalance_refresh_timeout_secs), jupiter_version: cli.jupiter_version.into(), - skip_tokens: cli.rebalance_skip_tokens.unwrap_or_default(), + skip_tokens: cli.rebalance_skip_tokens.clone().unwrap_or(Vec::new()), alternate_jupiter_route_tokens: cli .rebalance_alternate_jupiter_route_tokens + .clone() + .unwrap_or_default(), + alternate_sanctum_route_tokens: cli + .rebalance_alternate_sanctum_route_tokens + .clone() .unwrap_or_default(), - allow_withdraws: signer_is_owner, + allow_withdraws: cli.rebalance_using_limit_order == BoolArg::False || signer_is_owner, + use_sanctum: cli.sanctum_enabled == BoolArg::True, + use_limit_order: cli.rebalance_using_limit_order == BoolArg::True, + limit_order_distance_from_oracle_price_bps: cli + .rebalance_limit_order_distance_from_oracle_price_bps, }; rebalance_config.validate(&mango_client.context); - let rebalancer = Arc::new(rebalance::Rebalancer { + let mut rebalancer = rebalance::Rebalancer { mango_client: mango_client.clone(), account_fetcher: account_fetcher.clone(), mango_account_address: cli.liqor_mango_account, config: rebalance_config, + sanctum_supported_mints: HashSet::::new(), + }; + + let live_rpc_client = mango_client.client.new_rpc_async(); + rebalancer.init(&live_rpc_client).await; + let rebalancer = Arc::new(rebalancer); + + let liquidation = Box::new(LiquidationState { + mango_client: mango_client.clone(), + account_fetcher: account_fetcher.clone(), + liquidation_config: liq_config, + errors: Arc::new(RwLock::new( + ErrorTracking::builder() + .skip_threshold(2) + .skip_threshold_for_type(LiqErrorType::Liq, 5) + .skip_duration(Duration::from_secs(120)) + .build()?, + )), + oracle_errors: Arc::new(RwLock::new( + ErrorTracking::builder() + .skip_threshold(1) + .skip_duration(Duration::from_secs( + cli.skip_oracle_error_in_logs_duration_secs, + )) + .build()?, + )), }); - let mut liquidation = Box::new(LiquidationState { + let tcs = Box::new(TcsState { mango_client: mango_client.clone(), account_fetcher, - liquidation_config: liq_config, trigger_tcs_config: tcs_config, token_swap_info: token_swap_info_updater.clone(), - errors: ErrorTracking::builder() - .skip_threshold(2) - .skip_threshold_for_type(LiqErrorType::Liq, 5) - .skip_duration(Duration::from_secs(120)) - .build()?, - oracle_errors: ErrorTracking::builder() - .skip_threshold(1) - .skip_duration(Duration::from_secs( - cli.skip_oracle_error_in_logs_duration_secs, - )) - .build()?, + errors: Arc::new(RwLock::new( + ErrorTracking::builder() + .skip_threshold(2) + .skip_threshold_for_type(LiqErrorType::Liq, 5) + .skip_duration(Duration::from_secs(120)) + .build()?, + )), }); info!("main loop"); @@ -374,126 +421,87 @@ async fn main() -> anyhow::Result<()> { } }); + let mut optional_jobs = vec![]; + // Could be refactored to only start the below jobs when the first snapshot is done. // But need to take care to abort if the above job aborts beforehand. + if cli.rebalance == BoolArg::True { + let rebalance_job = + spawn_rebalance_job(shared_state.clone(), rebalance_trigger_receiver, rebalancer); + optional_jobs.push(rebalance_job); + } - let rebalance_job = tokio::spawn({ - let shared_state = shared_state.clone(); - async move { - loop { - tokio::select! { - _ = rebalance_interval.tick() => {} - _ = rebalance_trigger_receiver.recv() => {} - } - if !shared_state.read().unwrap().one_snapshot_done { - continue; - } - if let Err(err) = rebalancer.zero_all_non_quote().await { - error!("failed to rebalance liqor: {:?}", err); - - // Workaround: We really need a sequence enforcer in the liquidator since we don't want to - // accidentally send a similar tx again when we incorrectly believe an earlier one got forked - // off. For now, hard sleep on error to avoid the most frequent error cases. - tokio::time::sleep(Duration::from_secs(10)).await; - } - } - } - }); - - let liquidation_job = tokio::spawn({ - let mut interval = - mango_v4_client::delay_interval(Duration::from_millis(cli.check_interval_ms)); - let mut metric_liquidation_check = metrics.register_latency("liquidation_check".into()); - let mut metric_liquidation_start_end = - metrics.register_latency("liquidation_start_end".into()); - - let mut liquidation_start_time = None; - let mut tcs_start_time = None; - - let shared_state = shared_state.clone(); - async move { - loop { - interval.tick().await; - - let account_addresses = { - let mut state = shared_state.write().unwrap(); - if !state.one_snapshot_done { - // discard first latency info as it will skew data too much - state.oldest_chain_event_reception_time = None; - continue; - } - if state.oldest_chain_event_reception_time.is_none() - && liquidation_start_time.is_none() - { - // no new update, skip computing - continue; - } - - state.mango_accounts.iter().cloned().collect_vec() - }; - - liquidation.errors.update(); - liquidation.oracle_errors.update(); - - if liquidation_start_time.is_none() { - liquidation_start_time = Some(Instant::now()); - } - - let liquidated = liquidation - .maybe_liquidate_one(account_addresses.iter()) - .await; + if cli.liquidation_enabled == BoolArg::True { + let liquidation_job = liquidation_state::spawn_liquidation_job( + &cli, + &shared_state, + tx_liq_trigger_sender.clone(), + liquidation.clone(), + &metrics, + ); + optional_jobs.push(liquidation_job); + } - if !liquidated { - // This will be incorrect if we liquidate the last checked account - // (We will wait for next full run, skewing latency metrics) - // Probability is very low, might not need to be fixed + if cli.take_tcs == BoolArg::True { + let tcs_job = tcs_state::spawn_tcs_job( + &cli, + &shared_state, + tx_tcs_trigger_sender.clone(), + tcs.clone(), + &metrics, + ); + optional_jobs.push(tcs_job); + } - let mut state = shared_state.write().unwrap(); - let reception_time = state.oldest_chain_event_reception_time.unwrap(); - let current_time = Instant::now(); + if cli.liquidation_enabled == BoolArg::True || cli.take_tcs == BoolArg::True { + let mut tx_sender_jobs = tx_sender::spawn_tx_senders_job( + cli.max_parallel_operations, + cli.liquidation_enabled == BoolArg::True, + tx_liq_trigger_receiver, + tx_tcs_trigger_receiver, + tx_tcs_trigger_sender, + rebalance_trigger_sender, + shared_state.clone(), + liquidation, + tcs, + ); + optional_jobs.append(&mut tx_sender_jobs); + } - state.oldest_chain_event_reception_time = None; + if cli.telemetry == BoolArg::True { + optional_jobs.push(spawn_telemetry_job(&cli, mango_client.clone())); + } - metric_liquidation_check.push(current_time - reception_time); - metric_liquidation_start_end - .push(current_time - liquidation_start_time.unwrap()); - liquidation_start_time = None; - } + let token_swap_info_job = + spawn_token_swap_refresh_job(&cli, shared_state, token_swap_info_updater); + let check_changes_for_abort_job = spawn_context_change_watchdog_job(mango_client.clone()); - let mut took_tcs = false; - if !liquidated && cli.take_tcs == BoolArg::True { - tcs_start_time = Some(tcs_start_time.unwrap_or(Instant::now())); - - took_tcs = liquidation - .maybe_take_token_conditional_swap(account_addresses.iter()) - .await - .unwrap_or_else(|err| { - error!("error during maybe_take_token_conditional_swap: {err}"); - false - }); - - if !took_tcs { - let current_time = Instant::now(); - let mut metric_tcs_start_end = - metrics.register_latency("tcs_start_end".into()); - metric_tcs_start_end.push(current_time - tcs_start_time.unwrap()); - tcs_start_time = None; - } - } + let mut jobs: futures::stream::FuturesUnordered<_> = vec![ + snapshot_job, + data_job, + token_swap_info_job, + check_changes_for_abort_job, + ] + .into_iter() + .chain(optional_jobs) + .chain(prio_jobs.into_iter()) + .collect(); + jobs.next().await; - if liquidated || took_tcs { - rebalance_trigger_sender.send_unless_full(()).unwrap(); - } - } - } - }); + error!("a critical job aborted, exiting"); + Ok(()) +} - let token_swap_info_job = tokio::spawn({ +fn spawn_token_swap_refresh_job( + cli: &Cli, + shared_state: Arc>, + token_swap_info_updater: Arc, +) -> JoinHandle<()> { + tokio::spawn({ let mut interval = mango_v4_client::delay_interval(Duration::from_secs( cli.token_swap_refresh_interval_secs, )); let mut startup_wait = mango_v4_client::delay_interval(Duration::from_secs(1)); - let shared_state = shared_state.clone(); async move { loop { if !shared_state.read().unwrap().one_snapshot_done { @@ -517,41 +525,56 @@ async fn main() -> anyhow::Result<()> { token_swap_info_updater.log_all(); } } - }); + }) +} - let check_changes_for_abort_job = - tokio::spawn(MangoClient::loop_check_for_context_changes_and_abort( - mango_client.clone(), - Duration::from_secs(300), - )); +fn spawn_context_change_watchdog_job(mango_client: Arc) -> JoinHandle<()> { + tokio::spawn(MangoClient::loop_check_for_context_changes_and_abort( + mango_client, + Duration::from_secs(300), + )) +} - if cli.telemetry == BoolArg::True { - tokio::spawn(telemetry::report_regularly( - mango_client, - cli.min_health_ratio, - )); - } +fn spawn_telemetry_job(cli: &Cli, mango_client: Arc) -> JoinHandle<()> { + tokio::spawn(telemetry::report_regularly( + mango_client, + cli.min_health_ratio, + )) +} - use cli_args::{BoolArg, Cli, CliDotenv}; - use futures::StreamExt; - let mut jobs: futures::stream::FuturesUnordered<_> = vec![ - data_job, - rebalance_job, - liquidation_job, - token_swap_info_job, - check_changes_for_abort_job, - ] - .into_iter() - .chain(prio_jobs.into_iter()) - .collect(); - jobs.next().await; +fn spawn_rebalance_job( + shared_state: Arc>, + rebalance_trigger_receiver: async_channel::Receiver<()>, + rebalancer: Arc, +) -> JoinHandle<()> { + let mut rebalance_interval = tokio::time::interval(Duration::from_secs(30)); - error!("a critical job aborted, exiting"); - Ok(()) + tokio::spawn({ + async move { + loop { + tokio::select! { + _ = rebalance_interval.tick() => {} + _ = rebalance_trigger_receiver.recv() => {} + } + if !shared_state.read().unwrap().one_snapshot_done { + continue; + } + if let Err(err) = rebalancer.zero_all_non_quote().await { + error!("failed to rebalance liqor: {:?}", err); + + // TODO FAS Are there other scenario where this sleep is useful ? + // Workaround: We really need a sequence enforcer in the liquidator since we don't want to + // accidentally send a similar tx again when we incorrectly believe an earlier one got forked + // off. For now, hard sleep on error to avoid the most frequent error cases. + tokio::time::sleep(Duration::from_secs(10)).await; + } + } + } + }) } #[derive(Default)] -struct SharedState { +pub struct SharedState { /// Addresses of the MangoAccounts belonging to the mango program. /// Needed to check health of them all when the cache updates. mango_accounts: HashSet, @@ -561,6 +584,18 @@ struct SharedState { /// Oldest chain event not processed yet oldest_chain_event_reception_time: Option, + + /// Liquidation candidates (locally identified as liquidatable) + liquidation_candidates_accounts: indexmap::set::IndexSet, + + /// Interesting TCS that should be triggered + interesting_tcs: indexmap::set::IndexSet<(Pubkey, u64, u64)>, + + /// Liquidation currently being processed by a worker + processing_liquidation: HashSet, + + // TCS currently being processed by a worker + processing_tcs: HashSet<(Pubkey, u64, u64)>, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -584,218 +619,6 @@ impl std::fmt::Display for LiqErrorType { } } -struct LiquidationState { - mango_client: Arc, - account_fetcher: Arc, - token_swap_info: Arc, - liquidation_config: liquidate::Config, - trigger_tcs_config: trigger_tcs::Config, - - errors: ErrorTracking, - oracle_errors: ErrorTracking, -} - -impl LiquidationState { - async fn maybe_liquidate_one<'b>( - &mut self, - accounts_iter: impl Iterator, - ) -> bool { - use rand::seq::SliceRandom; - - let mut accounts = accounts_iter.collect::>(); - { - let mut rng = rand::thread_rng(); - accounts.shuffle(&mut rng); - } - - for pubkey in accounts { - if self - .maybe_liquidate_and_log_error(pubkey) - .await - .unwrap_or(false) - { - return true; - } - } - - false - } - - async fn maybe_liquidate_and_log_error(&mut self, pubkey: &Pubkey) -> anyhow::Result { - let now = Instant::now(); - let error_tracking = &mut self.errors; - - // Skip a pubkey if there've been too many errors recently - if let Some(error_entry) = - error_tracking.had_too_many_errors(LiqErrorType::Liq, pubkey, now) - { - trace!( - %pubkey, - error_entry.count, - "skip checking account for liquidation, had errors recently", - ); - return Ok(false); - } - - let result = liquidate::maybe_liquidate_account( - &self.mango_client, - &self.account_fetcher, - pubkey, - &self.liquidation_config, - ) - .await; - - if let Err(err) = result.as_ref() { - if let Some((ti, ti_name)) = err.try_unwrap_oracle_error() { - if self - .oracle_errors - .had_too_many_errors(LiqErrorType::Liq, &ti, Instant::now()) - .is_none() - { - warn!( - "{:?} recording oracle error for token {} {}", - chrono::offset::Utc::now(), - ti_name, - ti - ); - } - - self.oracle_errors - .record(LiqErrorType::Liq, &ti, err.to_string()); - return result; - } - - // Keep track of pubkeys that had errors - error_tracking.record(LiqErrorType::Liq, pubkey, err.to_string()); - - // Not all errors need to be raised to the user's attention. - let mut is_error = true; - - // Simulation errors due to liqee precondition failures on the liquidation instructions - // will commonly happen if our liquidator is late or if there are chain forks. - match err.downcast_ref::() { - Some(MangoClientError::SendTransactionPreflightFailure { logs, .. }) => { - if logs.iter().any(|line| { - line.contains("HealthMustBeNegative") || line.contains("IsNotBankrupt") - }) { - is_error = false; - } - } - _ => {} - }; - if is_error { - error!("liquidating account {}: {:?}", pubkey, err); - } else { - trace!("liquidating account {}: {:?}", pubkey, err); - } - } else { - error_tracking.clear(LiqErrorType::Liq, pubkey); - } - - result - } - - async fn maybe_take_token_conditional_swap( - &mut self, - accounts_iter: impl Iterator, - ) -> anyhow::Result { - let accounts = accounts_iter.collect::>(); - - let now = Instant::now(); - let now_ts: u64 = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH)? - .as_secs(); - - let tcs_context = trigger_tcs::Context { - mango_client: self.mango_client.clone(), - account_fetcher: self.account_fetcher.clone(), - token_swap_info: self.token_swap_info.clone(), - config: self.trigger_tcs_config.clone(), - jupiter_quote_cache: Arc::new(trigger_tcs::JupiterQuoteCache::default()), - now_ts, - }; - - // Find interesting (pubkey, tcsid, volume) - let mut interesting_tcs = Vec::with_capacity(accounts.len()); - for pubkey in accounts.iter() { - if let Some(error_entry) = - self.errors - .had_too_many_errors(LiqErrorType::TcsCollectionHard, pubkey, now) - { - trace!( - %pubkey, - error_entry.count, - "skip checking account for tcs, had errors recently", - ); - continue; - } - - match tcs_context.find_interesting_tcs_for_account(pubkey) { - Ok(v) => { - self.errors.clear(LiqErrorType::TcsCollectionHard, pubkey); - if v.is_empty() { - self.errors - .clear(LiqErrorType::TcsCollectionPartial, pubkey); - self.errors.clear(LiqErrorType::TcsExecution, pubkey); - } else if v.iter().all(|it| it.is_ok()) { - self.errors - .clear(LiqErrorType::TcsCollectionPartial, pubkey); - } else { - for it in v.iter() { - if let Err(e) = it { - self.errors.record( - LiqErrorType::TcsCollectionPartial, - pubkey, - e.to_string(), - ); - } - } - } - interesting_tcs.extend(v.iter().filter_map(|it| it.as_ref().ok())); - } - Err(e) => { - self.errors - .record(LiqErrorType::TcsCollectionHard, pubkey, e.to_string()); - } - } - } - if interesting_tcs.is_empty() { - return Ok(false); - } - - let (txsigs, mut changed_pubkeys) = tcs_context - .execute_tcs(&mut interesting_tcs, &mut self.errors) - .await?; - for pubkey in changed_pubkeys.iter() { - self.errors.clear(LiqErrorType::TcsExecution, pubkey); - } - if txsigs.is_empty() { - return Ok(false); - } - changed_pubkeys.push(self.mango_client.mango_account_address); - - // Force a refresh of affected accounts - let slot = self - .account_fetcher - .transaction_max_slot(&txsigs) - .await - .context("transaction_max_slot")?; - if let Err(e) = self - .account_fetcher - .refresh_accounts_via_rpc_until_slot( - &changed_pubkeys, - slot, - self.liquidation_config.refresh_timeout, - ) - .await - { - info!(slot, "could not refresh after tcs execution: {}", e); - } - - Ok(true) - } -} - fn start_chain_data_metrics(chain: Arc>, metrics: &metrics::Metrics) { let mut interval = mango_v4_client::delay_interval(Duration::from_secs(600)); diff --git a/bin/liquidator/src/rebalance.rs b/bin/liquidator/src/rebalance.rs index fbcd28f29b..6211065821 100644 --- a/bin/liquidator/src/rebalance.rs +++ b/bin/liquidator/src/rebalance.rs @@ -2,18 +2,27 @@ use itertools::Itertools; use mango_v4::accounts_zerocopy::KeyedAccountSharedData; use mango_v4::state::{ Bank, BookSide, MangoAccountValue, OracleAccountInfos, PerpMarket, PerpPosition, - PlaceOrderType, Side, TokenIndex, QUOTE_TOKEN_INDEX, + PlaceOrderType, Serum3MarketIndex, Side, TokenIndex, QUOTE_TOKEN_INDEX, }; use mango_v4_client::{ - chain_data, jupiter, perp_pnl, MangoClient, MangoGroupContext, PerpMarketContext, TokenContext, - TransactionBuilder, TransactionSize, + chain_data, perp_pnl, swap, MangoClient, MangoGroupContext, PerpMarketContext, + PreparedInstructions, Serum3MarketContext, TokenContext, TransactionBuilder, TransactionSize, }; +use solana_client::nonblocking::rpc_client::RpcClient; use {fixed::types::I80F48, solana_sdk::pubkey::Pubkey}; +use fixed::types::extra::U48; +use fixed::FixedI128; +use mango_v4::accounts_ix::{Serum3OrderType, Serum3SelfTradeBehavior, Serum3Side}; +use mango_v4::serum3_cpi; +use mango_v4::serum3_cpi::{OpenOrdersAmounts, OpenOrdersSlim}; +use solana_sdk::account::ReadableAccount; use solana_sdk::signature::Signature; +use std::collections::{HashMap, HashSet}; +use std::future::Future; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tracing::*; #[derive(Clone)] @@ -21,15 +30,20 @@ pub struct Config { pub enabled: bool, /// Maximum slippage allowed in Jupiter pub slippage_bps: u64, + /// Maximum slippage from oracle price for limit orders + pub limit_order_distance_from_oracle_price_bps: u64, /// When closing borrows, the rebalancer can't close token positions exactly. /// Instead it purchases too much and then gets rid of the excess in a second step. /// If this is 1.05, then it'll swap borrow_value * 1.05 quote token into borrow token. pub borrow_settle_excess: f64, pub refresh_timeout: Duration, - pub jupiter_version: jupiter::Version, + pub jupiter_version: swap::Version, pub skip_tokens: Vec, pub alternate_jupiter_route_tokens: Vec, + pub alternate_sanctum_route_tokens: Vec, pub allow_withdraws: bool, + pub use_sanctum: bool, + pub use_limit_order: bool, } impl Config { @@ -56,6 +70,7 @@ pub struct Rebalancer { pub account_fetcher: Arc, pub mango_account_address: Pubkey, pub config: Config, + pub sanctum_supported_mints: HashSet, } impl Rebalancer { @@ -69,9 +84,19 @@ impl Rebalancer { "checking for rebalance" ); - self.rebalance_perps().await?; - self.rebalance_tokens().await?; + let rebalance_perps_res = self.rebalance_perps().await; + let rebalance_tokens_res = self.rebalance_tokens().await; + if rebalance_perps_res.is_err() && rebalance_tokens_res.is_err() { + anyhow::bail!( + "Failed to rebalance perps ({}) and tokens ({})", + rebalance_perps_res.unwrap_err(), + rebalance_tokens_res.unwrap_err() + ) + } + + rebalance_perps_res?; + rebalance_tokens_res?; Ok(()) } @@ -95,16 +120,16 @@ impl Rebalancer { Ok(true) } - async fn jupiter_quote( + async fn swap_quote( &self, input_mint: Pubkey, output_mint: Pubkey, amount: u64, only_direct_routes: bool, - jupiter_version: jupiter::Version, - ) -> anyhow::Result { + jupiter_version: swap::Version, + ) -> anyhow::Result { self.mango_client - .jupiter() + .swap() .quote( input_mint, output_mint, @@ -116,28 +141,31 @@ impl Rebalancer { .await } - /// Grab three possible routes: + /// Grab multiples possible routes: /// 1. USDC -> output (complex routes) /// 2. USDC -> output (direct route only) - /// 3. alternate_jupiter_route_tokens -> output (direct route only) - /// Use 1. if it fits into a tx. Otherwise use the better of 2./3. + /// 3. if enabled, sanctum routes - might generate 0, 1 or more routes + /// 4. input -> alternate_jupiter_route_tokens (direct route only) - might generate 0, 1 or more routes + /// Use best of 1/2/3. if it fits into a tx, + /// Otherwise use the best of 4. async fn token_swap_buy( &self, + account: &MangoAccountValue, output_mint: Pubkey, in_amount_quote: u64, - ) -> anyhow::Result<(Signature, jupiter::Quote)> { + ) -> anyhow::Result<(Signature, swap::Quote)> { let quote_token = self.mango_client.context.token(QUOTE_TOKEN_INDEX); let quote_mint = quote_token.mint; let jupiter_version = self.config.jupiter_version; - let full_route_job = self.jupiter_quote( + let full_route_job = self.swap_quote( quote_mint, output_mint, in_amount_quote, false, jupiter_version, ); - let direct_quote_route_job = self.jupiter_quote( + let direct_quote_route_job = self.swap_quote( quote_mint, output_mint, in_amount_quote, @@ -146,75 +174,137 @@ impl Rebalancer { ); let mut jobs = vec![full_route_job, direct_quote_route_job]; - for in_token_index in &self.config.alternate_jupiter_route_tokens { - let in_token = self.mango_client.context.token(*in_token_index); - // For the alternate output routes we need to adjust the in amount by the token price - let in_price = self - .account_fetcher - .fetch_bank_price(&in_token.first_bank())?; - let in_amount = (I80F48::from(in_amount_quote) / in_price) - .ceil() - .to_num::(); - let direct_route_job = - self.jupiter_quote(in_token.mint, output_mint, in_amount, true, jupiter_version); - jobs.push(direct_route_job); + if self.can_use_sanctum_for_token(output_mint)? { + for in_token_index in &self.config.alternate_sanctum_route_tokens { + let (alt_mint, alt_in_amount) = + self.get_alternative_token_amount(in_token_index, in_amount_quote)?; + let sanctum_alt_route_job = self.swap_quote( + alt_mint, + output_mint, + alt_in_amount, + false, + swap::Version::Sanctum, + ); + jobs.push(sanctum_alt_route_job); + } } - let mut results = futures::future::join_all(jobs).await; - let full_route = results.remove(0)?; - let alternatives = results.into_iter().filter_map(|v| v.ok()).collect_vec(); + let results = futures::future::join_all(jobs).await; + let routes: Vec<_> = results.into_iter().filter_map(|v| v.ok()).collect_vec(); - let (tx_builder, route) = self - .determine_best_jupiter_tx( - // If the best_route couldn't be fetched, something is wrong - &full_route, - &alternatives, - ) + let best_route_res = self + .determine_best_swap_tx(routes, quote_mint, output_mint) + .await; + + let (mut tx_builder, route) = match best_route_res { + Ok(x) => x, + Err(e) => { + warn!("could not use simple routes because of {}, trying with an alternative one (if configured)", e); + + self.get_jupiter_alt_route( + |in_token_index| { + let (alt_mint, alt_in_amount) = + self.get_alternative_token_amount(in_token_index, in_amount_quote)?; + let swap = self.swap_quote( + alt_mint, + output_mint, + alt_in_amount, + true, + jupiter_version, + ); + Ok(swap) + }, + quote_mint, + output_mint, + ) + .await? + } + }; + + let seq_check_ix = self + .mango_client + .sequence_check_instruction(&self.mango_account_address, account) .await?; + tx_builder.append(seq_check_ix); + let sig = tx_builder .send_and_confirm(&self.mango_client.client) .await?; Ok((sig, route)) } - /// Grab three possible routes: + /// Grab multiples possible routes: /// 1. input -> USDC (complex routes) /// 2. input -> USDC (direct route only) - /// 3. input -> alternate_jupiter_route_tokens (direct route only) - /// Use 1. if it fits into a tx. Otherwise use the better of 2./3. + /// 3. if enabled, sanctum routes - might generate 0, 1 or more routes + /// 4. input -> alternate_jupiter_route_tokens (direct route only) - might generate 0, 1 or more routes + /// Use best of 1/2/3. if it fits into a tx, + /// Otherwise use the best of 4. async fn token_swap_sell( &self, + account: &MangoAccountValue, input_mint: Pubkey, in_amount: u64, - ) -> anyhow::Result<(Signature, jupiter::Quote)> { + ) -> anyhow::Result<(Signature, swap::Quote)> { let quote_token = self.mango_client.context.token(QUOTE_TOKEN_INDEX); let quote_mint = quote_token.mint; let jupiter_version = self.config.jupiter_version; let full_route_job = - self.jupiter_quote(input_mint, quote_mint, in_amount, false, jupiter_version); + self.swap_quote(input_mint, quote_mint, in_amount, false, jupiter_version); let direct_quote_route_job = - self.jupiter_quote(input_mint, quote_mint, in_amount, true, jupiter_version); + self.swap_quote(input_mint, quote_mint, in_amount, true, jupiter_version); let mut jobs = vec![full_route_job, direct_quote_route_job]; - for out_token_index in &self.config.alternate_jupiter_route_tokens { - let out_token = self.mango_client.context.token(*out_token_index); - let direct_route_job = - self.jupiter_quote(input_mint, out_token.mint, in_amount, true, jupiter_version); - jobs.push(direct_route_job); + if self.can_use_sanctum_for_token(input_mint)? { + for out_token_index in &self.config.alternate_sanctum_route_tokens { + let out_token = self.mango_client.context.token(*out_token_index); + let sanctum_job = self.swap_quote( + input_mint, + out_token.mint, + in_amount, + false, + swap::Version::Sanctum, + ); + jobs.push(sanctum_job); + } } - let mut results = futures::future::join_all(jobs).await; - let full_route = results.remove(0)?; - let alternatives = results.into_iter().filter_map(|v| v.ok()).collect_vec(); + let results = futures::future::join_all(jobs).await; + let routes: Vec<_> = results.into_iter().filter_map(|v| v.ok()).collect_vec(); - let (tx_builder, route) = self - .determine_best_jupiter_tx( - // If the best_route couldn't be fetched, something is wrong - &full_route, - &alternatives, - ) + let best_route_res = self + .determine_best_swap_tx(routes, input_mint, quote_mint) + .await; + + let (mut tx_builder, route) = match best_route_res { + Ok(x) => x, + Err(e) => { + warn!("could not use simple routes because of {}, trying with an alternative one (if configured)", e); + + self.get_jupiter_alt_route( + |out_token_index| { + let out_token = self.mango_client.context.token(*out_token_index); + Ok(self.swap_quote( + input_mint, + out_token.mint, + in_amount, + true, + jupiter_version, + )) + }, + input_mint, + quote_mint, + ) + .await? + } + }; + + let seq_check_ix = self + .mango_client + .sequence_check_instruction(&self.mango_account_address, account) .await?; + tx_builder.append(seq_check_ix); let sig = tx_builder .send_and_confirm(&self.mango_client.client) @@ -222,47 +312,133 @@ impl Rebalancer { Ok((sig, route)) } - async fn determine_best_jupiter_tx( + fn get_alternative_token_amount( &self, - full: &jupiter::Quote, - alternatives: &[jupiter::Quote], - ) -> anyhow::Result<(TransactionBuilder, jupiter::Quote)> { - let builder = self - .mango_client - .jupiter() - .prepare_swap_transaction(full) - .await?; - let tx_size = builder.transaction_size()?; - if tx_size.is_within_limit() { - return Ok((builder, full.clone())); + in_token_index: &u16, + in_amount_quote: u64, + ) -> anyhow::Result<(Pubkey, u64)> { + let in_token: &TokenContext = self.mango_client.context.token(*in_token_index); + let in_price = self + .account_fetcher + .fetch_bank_price(&in_token.first_bank())?; + let in_amount = (I80F48::from(in_amount_quote) / in_price) + .ceil() + .to_num::(); + + Ok((in_token.mint, in_amount)) + } + + fn can_use_sanctum_for_token(&self, mint: Pubkey) -> anyhow::Result { + if !self.config.use_sanctum { + return Ok(false); } - trace!( - route_label = full.first_route_label(), - %full.input_mint, - %full.output_mint, - ?tx_size, - limit = ?TransactionSize::limit(), - "full route does not fit in a tx", - ); - if alternatives.is_empty() { - anyhow::bail!( - "no alternative routes from {} to {}", - full.input_mint, - full.output_mint - ); + let token = self.mango_client.context.token_by_mint(&mint)?; + + let can_swap_on_sanctum = self.can_swap_on_sanctum(mint); + + // forbid swapping to something that could be used in another sanctum swap, creating a cycle + let is_an_alt_for_sanctum = self + .config + .alternate_sanctum_route_tokens + .contains(&token.token_index); + + Ok(can_swap_on_sanctum && !is_an_alt_for_sanctum) + } + + async fn get_jupiter_alt_route>>( + &self, + quote_fetcher: impl Fn(&u16) -> anyhow::Result, + original_input_mint: Pubkey, + original_output_mint: Pubkey, + ) -> anyhow::Result<(TransactionBuilder, swap::Quote)> { + let mut alt_jobs = vec![]; + for in_token_index in &self.config.alternate_jupiter_route_tokens { + alt_jobs.push(quote_fetcher(in_token_index)?); } + let alt_results = futures::future::join_all(alt_jobs).await; + let alt_routes: Vec<_> = alt_results.into_iter().filter_map(|v| v.ok()).collect_vec(); - let best = alternatives - .iter() - .min_by(|a, b| a.price_impact_pct.partial_cmp(&b.price_impact_pct).unwrap()) - .unwrap(); - let builder = self - .mango_client - .jupiter() - .prepare_swap_transaction(best) + let best_route = self + .determine_best_swap_tx(alt_routes, original_input_mint, original_output_mint) .await?; - Ok((builder, best.clone())) + Ok(best_route) + } + + async fn determine_best_swap_tx( + &self, + mut routes: Vec, + original_input_mint: Pubkey, + original_output_mint: Pubkey, + ) -> anyhow::Result<(TransactionBuilder, swap::Quote)> { + let mut prices = HashMap::::new(); + let mut get_or_fetch_price = |m| { + let entry = prices.entry(m).or_insert_with(|| { + let token = self + .mango_client + .context + .token_by_mint(&m) + .expect("token for mint not found"); + self.account_fetcher + .fetch_bank_price(&token.first_bank()) + .expect("failed to fetch price") + }); + *entry + }; + + routes.sort_by_cached_key(|r| { + let in_price = get_or_fetch_price(r.input_mint); + let out_price = get_or_fetch_price(r.output_mint); + let amount = out_price * I80F48::from_num(r.out_amount) + - in_price * I80F48::from_num(r.in_amount); + + let t = match r.raw { + swap::RawQuote::Mock => "mock", + swap::RawQuote::V6(_) => "jupiter", + swap::RawQuote::Sanctum(_) => "sanctum", + }; + + debug!( + "quote for {} vs {} [using {}] is {}@{} vs {}@{} -> amount={}", + r.input_mint, + r.output_mint, + t, + r.in_amount, + in_price, + r.out_amount, + out_price, + amount + ); + + std::cmp::Reverse(amount) + }); + + for route in routes { + let builder = self + .mango_client + .swap() + .prepare_swap_transaction(&route) + .await?; + let tx_size = builder.transaction_size()?; + if tx_size.is_within_limit() { + return Ok((builder, route.clone())); + } + + trace!( + route_label = route.first_route_label(), + %route.input_mint, + %route.output_mint, + ?tx_size, + limit = ?TransactionSize::limit(), + "route does not fit in a tx", + ); + } + + anyhow::bail!( + "no routes from {} to {}", + original_input_mint, + original_output_mint + ); } fn mango_account(&self) -> anyhow::Result> { @@ -273,12 +449,13 @@ impl Rebalancer { } async fn rebalance_tokens(&self) -> anyhow::Result<()> { + self.close_and_settle_all_openbook_orders().await?; let account = self.mango_account()?; // TODO: configurable? let quote_token = self.mango_client.context.token(QUOTE_TOKEN_INDEX); - for token_position in account.active_token_positions() { + for token_position in Self::shuffle(account.active_token_positions()) { let token_index = token_position.token_index; let token = self.mango_client.context.token(token_index); if token_index == quote_token.token_index @@ -299,7 +476,20 @@ impl Rebalancer { // to sell them. Instead they will be withdrawn at the end. // Purchases will aim to purchase slightly more than is needed, such that we can // again withdraw the dust at the end. - let dust_threshold = I80F48::from(2) / token_price; + let dust_threshold_res = if self.config.use_limit_order { + self.dust_threshold_for_limit_order(token) + .await + .map(|x| I80F48::from(x)) + } else { + Ok(I80F48::from(2) / token_price) + }; + + let Ok(dust_threshold) = dust_threshold_res + else { + let e = dust_threshold_res.unwrap_err(); + error!("Cannot rebalance token {}, probably missing USDC market ? - error: {}", token.name, e); + continue; + }; // Some rebalancing can actually change non-USDC positions (rebalancing to SOL) // So re-fetch the current token position amount @@ -313,57 +503,29 @@ impl Rebalancer { }; let mut amount = fresh_amount()?; - trace!(token_index, %amount, %dust_threshold, "checking"); - if amount < 0 { - // Buy - let buy_amount = - amount.abs().ceil() + (dust_threshold - I80F48::ONE).max(I80F48::ZERO); - let input_amount = - buy_amount * token_price * I80F48::from_num(self.config.borrow_settle_excess); - let (txsig, route) = self - .token_swap_buy(token_mint, input_amount.to_num()) - .await?; - let in_token = self - .mango_client - .context - .token_by_mint(&route.input_mint) - .unwrap(); - info!( - %txsig, - "bought {} {} for {} {}", - token.native_to_ui(I80F48::from(route.out_amount)), - token.name, - in_token.native_to_ui(I80F48::from(route.in_amount)), - in_token.name, - ); - if !self.refresh_mango_account_after_tx(txsig).await? { - return Ok(()); - } - amount = fresh_amount()?; - } + trace!(token_index, token.name, %amount, %dust_threshold, "checking"); - if amount > dust_threshold { - // Sell - let (txsig, route) = self - .token_swap_sell(token_mint, amount.to_num::()) + if self.config.use_limit_order { + self.unwind_using_limit_orders( + &account, + token, + token_price, + dust_threshold, + amount, + ) + .await?; + } else { + amount = self + .unwind_using_swap( + &account, + token, + token_mint, + token_price, + dust_threshold, + fresh_amount, + amount, + ) .await?; - let out_token = self - .mango_client - .context - .token_by_mint(&route.output_mint) - .unwrap(); - info!( - %txsig, - "sold {} {} for {} {}", - token.native_to_ui(I80F48::from(route.in_amount)), - token.name, - out_token.native_to_ui(I80F48::from(route.out_amount)), - out_token.name, - ); - if !self.refresh_mango_account_after_tx(txsig).await? { - return Ok(()); - } - amount = fresh_amount()?; } // Any remainder that could not be sold just gets withdrawn to ensure the @@ -398,6 +560,288 @@ impl Rebalancer { Ok(()) } + async fn dust_threshold_for_limit_order(&self, token: &TokenContext) -> anyhow::Result { + let (_, market) = self + .mango_client + .context + .serum3_markets + .iter() + .find(|(_, context)| { + context.base_token_index == token.token_index + && context.quote_token_index == QUOTE_TOKEN_INDEX + }) + .ok_or(anyhow::format_err!( + "could not find market for token {}", + token.name + ))?; + + Ok(market.coin_lot_size - 1) + } + + async fn unwind_using_limit_orders( + &self, + account: &Box, + token: &TokenContext, + token_price: I80F48, + dust_threshold: FixedI128, + native_amount: I80F48, + ) -> anyhow::Result<()> { + if native_amount >= 0 && native_amount < dust_threshold { + return Ok(()); + } + + let (market_index, market) = self + .mango_client + .context + .serum3_markets + .iter() + .find(|(_, context)| { + context.base_token_index == token.token_index + && context.quote_token_index == QUOTE_TOKEN_INDEX + }) + .ok_or(anyhow::format_err!( + "could not find market for token {}", + token.name + ))?; + + let side = if native_amount < 0 { + Serum3Side::Bid + } else { + Serum3Side::Ask + }; + + let distance_from_oracle_price_bp = + I80F48::from_num(self.config.limit_order_distance_from_oracle_price_bps) + * match side { + Serum3Side::Bid => 1, + Serum3Side::Ask => -1, + }; + let price_adjustment_factor = + (I80F48::from_num(10_000) + distance_from_oracle_price_bp) / I80F48::from_num(10_000); + + let limit_price = + (token_price * price_adjustment_factor * I80F48::from_num(market.coin_lot_size)) + .to_num::() + / market.pc_lot_size; + let mut max_base_lots = + (native_amount.abs() / I80F48::from_num(market.coin_lot_size)).to_num::(); + + debug!( + side = match side { + Serum3Side::Bid => "Buy", + Serum3Side::Ask => "Sell", + }, + token = token.name, + oracle_price = token_price.to_num::(), + price_adjustment_factor = price_adjustment_factor.to_num::(), + coin_lot_size = market.coin_lot_size, + pc_lot_size = market.pc_lot_size, + limit_price, + native_amount = native_amount.to_num::(), + max_base_lots = max_base_lots, + "building order for rebalancing" + ); + + // Try to buy enough to close the borrow + if max_base_lots == 0 && native_amount < 0 { + info!( + "Buying a whole lot for token {} to cover borrow of {}", + token.name, native_amount + ); + max_base_lots = 1; + } + + if max_base_lots == 0 { + warn!("Could not rebalance token '{}' (native_amount={}) using limit order, below base lot size", token.name, native_amount); + return Ok(()); + } + + let mut account = account.clone(); + let create_or_replace_account_ixs = self + .mango_client + .serum3_create_or_replace_account_instruction(&mut account, *market_index, side) + .await?; + let cancel_ixs = + self.mango_client + .serum3_cancel_all_orders_instruction(&account, *market_index, 10)?; + let place_order_ixs = self + .mango_client + .serum3_place_order_instruction( + &account, + *market_index, + side, + limit_price, + max_base_lots, + ((limit_price * max_base_lots * market.pc_lot_size) as f64 * 1.01) as u64, + Serum3SelfTradeBehavior::CancelProvide, + Serum3OrderType::Limit, + SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis() as u64, + 10, + ) + .await?; + + let seq_check_ixs = self + .mango_client + .sequence_check_instruction(&self.mango_account_address, &account) + .await?; + + let mut ixs = PreparedInstructions::new(); + ixs.append(create_or_replace_account_ixs); + ixs.append(cancel_ixs); + ixs.append(place_order_ixs); + ixs.append(seq_check_ixs); + + let txsig = self + .mango_client + .send_and_confirm_owner_tx(ixs.to_instructions()) + .await?; + + info!( + %txsig, + "placed order for {} {} at price = {}", + token.native_to_ui(I80F48::from(native_amount)), + token.name, + limit_price, + ); + if !self.refresh_mango_account_after_tx(txsig).await? { + return Ok(()); + } + + Ok(()) + } + + async fn close_and_settle_all_openbook_orders(&self) -> anyhow::Result<()> { + let account = self.mango_account()?; + + for x in Self::shuffle(account.active_serum3_orders()) { + let token = self.mango_client.context.token(x.base_token_index); + let quote = self.mango_client.context.token(x.quote_token_index); + let market_index = x.market_index; + let market = self + .mango_client + .context + .serum3_markets + .get(&market_index) + .expect("no openbook market found"); + self.close_and_settle_openbook_orders(&account, token, &market_index, market, quote) + .await?; + } + Ok(()) + } + + /// This will only settle funds when there is no more active orders (avoid doing too many settle tx) + async fn close_and_settle_openbook_orders( + &self, + account: &Box, + token: &TokenContext, + market_index: &Serum3MarketIndex, + market: &Serum3MarketContext, + quote: &TokenContext, + ) -> anyhow::Result<()> { + let Ok(open_orders) = account.serum3_orders(*market_index).map(|x| x.open_orders) + else { + return Ok(()); + }; + + let oo_acc = self.account_fetcher.fetch_raw(&open_orders)?; + let oo = serum3_cpi::load_open_orders_bytes(oo_acc.data())?; + let oo_slim = OpenOrdersSlim::from_oo(oo); + + if oo_slim.native_base_reserved() != 0 || oo_slim.native_quote_reserved() != 0 { + return Ok(()); + } + + let settle_ixs = + self.mango_client + .serum3_settle_funds_instruction(market, token, quote, open_orders); + + let close_ixs = self + .mango_client + .serum3_close_open_orders_instruction(*market_index); + + let mut ixs = PreparedInstructions::new(); + ixs.append(close_ixs); + ixs.append(settle_ixs); + + let txsig = self + .mango_client + .send_and_confirm_owner_tx(ixs.to_instructions()) + .await?; + + info!( + %txsig, + "settle spot funds for {}", + token.name, + ); + if !self.refresh_mango_account_after_tx(txsig).await? { + return Ok(()); + } + + Ok(()) + } + + async fn unwind_using_swap( + &self, + account: &Box, + token: &TokenContext, + token_mint: Pubkey, + token_price: I80F48, + dust_threshold: FixedI128, + fresh_amount: impl Fn() -> anyhow::Result, + amount: I80F48, + ) -> anyhow::Result { + if amount < 0 { + // Buy + let buy_amount = amount.abs().ceil() + (dust_threshold - I80F48::ONE).max(I80F48::ZERO); + let input_amount = + buy_amount * token_price * I80F48::from_num(self.config.borrow_settle_excess); + let (txsig, route) = self + .token_swap_buy(&account, token_mint, input_amount.to_num()) + .await?; + let in_token = self + .mango_client + .context + .token_by_mint(&route.input_mint) + .unwrap(); + info!( + %txsig, + "bought {} {} for {} {}", + token.native_to_ui(I80F48::from(route.out_amount)), + token.name, + in_token.native_to_ui(I80F48::from(route.in_amount)), + in_token.name, + ); + if !self.refresh_mango_account_after_tx(txsig).await? { + return Ok(amount); + } + } + + if amount > dust_threshold { + // Sell + let (txsig, route) = self + .token_swap_sell(&account, token_mint, amount.to_num::()) + .await?; + let out_token = self + .mango_client + .context + .token_by_mint(&route.output_mint) + .unwrap(); + info!( + %txsig, + "sold {} {} for {} {}", + token.native_to_ui(I80F48::from(route.in_amount)), + token.name, + out_token.native_to_ui(I80F48::from(route.out_amount)), + out_token.name, + ); + if !self.refresh_mango_account_after_tx(txsig).await? { + return Ok(amount); + } + } + + Ok(fresh_amount()?) + } + #[instrument( skip_all, fields( @@ -467,9 +911,10 @@ impl Rebalancer { return Ok(true); } - let txsig = self + let mut ixs = self .mango_client - .perp_place_order( + .perp_place_order_instruction( + account, perp_position.market_index, side, price_lots, @@ -483,6 +928,23 @@ impl Rebalancer { mango_v4::state::SelfTradeBehavior::DecrementTake, ) .await?; + + let seq_check_ix = self + .mango_client + .sequence_check_instruction(&self.mango_account_address, account) + .await?; + ixs.append(seq_check_ix); + + let tx_builder = TransactionBuilder { + instructions: ixs.to_instructions(), + signers: vec![self.mango_client.owner.clone()], + ..self.mango_client.transaction_builder().await? + }; + + let txsig = tx_builder + .send_and_confirm(&self.mango_client.client) + .await?; + info!( %txsig, %order_price, @@ -556,7 +1018,7 @@ impl Rebalancer { async fn rebalance_perps(&self) -> anyhow::Result<()> { let account = self.mango_account()?; - for perp_position in account.active_perp_positions() { + for perp_position in Self::shuffle(account.active_perp_positions()) { let perp = self.mango_client.context.perp(perp_position.market_index); if !self.rebalance_perp(&account, perp, perp_position).await? { return Ok(()); @@ -565,4 +1027,27 @@ impl Rebalancer { Ok(()) } + + fn shuffle(iterator: impl Iterator) -> Vec { + use rand::seq::SliceRandom; + + let mut result = iterator.collect::>(); + { + let mut rng = rand::thread_rng(); + result.shuffle(&mut rng); + } + + result + } + + fn can_swap_on_sanctum(&self, mint: Pubkey) -> bool { + self.sanctum_supported_mints.contains(&mint) + } + + pub async fn init(&mut self, live_rpc_client: &RpcClient) { + match swap::sanctum::load_supported_token_mints(live_rpc_client).await { + Err(e) => warn!("Could not load list of sanctum supported mint: {}", e), + Ok(mint) => self.sanctum_supported_mints.extend(mint), + } + } } diff --git a/bin/liquidator/src/tcs_state.rs b/bin/liquidator/src/tcs_state.rs new file mode 100644 index 0000000000..6434a212e4 --- /dev/null +++ b/bin/liquidator/src/tcs_state.rs @@ -0,0 +1,218 @@ +use crate::cli_args::Cli; +use crate::metrics::Metrics; +use crate::token_swap_info::TokenSwapInfoUpdater; +use crate::{trigger_tcs, LiqErrorType, SharedState}; +use anchor_lang::prelude::Pubkey; +use anyhow::Context; +use itertools::Itertools; +use mango_v4_client::error_tracking::ErrorTracking; +use mango_v4_client::{chain_data, MangoClient}; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; +use tokio::task::JoinHandle; +use tracing::{error, info, trace}; + +pub fn spawn_tcs_job( + cli: &Cli, + shared_state: &Arc>, + tx_trigger_sender: async_channel::Sender<()>, + mut tcs: Box, + metrics: &Metrics, +) -> JoinHandle<()> { + tokio::spawn({ + let mut interval = + mango_v4_client::delay_interval(Duration::from_millis(cli.tcs_check_interval_ms)); + let mut tcs_start_time = None; + let mut metric_tcs_start_end = metrics.register_latency("tcs_start_end".into()); + let shared_state = shared_state.clone(); + + async move { + loop { + interval.tick().await; + + let account_addresses = { + let state = shared_state.write().unwrap(); + if !state.one_snapshot_done { + continue; + } + state.mango_accounts.iter().cloned().collect_vec() + }; + + tcs.errors.write().unwrap().update(); + + tcs_start_time = Some(tcs_start_time.unwrap_or(Instant::now())); + + let found_candidates = tcs + .find_candidates(account_addresses.iter(), |candidate| { + if shared_state + .write() + .unwrap() + .interesting_tcs + .insert(candidate) + { + tx_trigger_sender.try_send(())?; + } + + Ok(()) + }) + .await + .unwrap_or_else(|err| { + error!("error during find_candidate: {err}"); + 0 + }); + + if found_candidates > 0 { + tracing::debug!("found {} candidates for triggering", found_candidates); + } + + let current_time = Instant::now(); + metric_tcs_start_end.push(current_time - tcs_start_time.unwrap()); + tcs_start_time = None; + } + } + }) +} + +#[derive(Clone)] +pub struct TcsState { + pub mango_client: Arc, + pub account_fetcher: Arc, + pub token_swap_info: Arc, + pub trigger_tcs_config: trigger_tcs::Config, + + pub errors: Arc>>, +} + +impl TcsState { + async fn find_candidates( + &mut self, + accounts_iter: impl Iterator, + action: impl Fn((Pubkey, u64, u64)) -> anyhow::Result<()>, + ) -> anyhow::Result { + let accounts = accounts_iter.collect::>(); + + let now = Instant::now(); + let now_ts: u64 = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_secs(); + + let tcs_context = trigger_tcs::Context { + mango_client: self.mango_client.clone(), + account_fetcher: self.account_fetcher.clone(), + token_swap_info: self.token_swap_info.clone(), + config: self.trigger_tcs_config.clone(), + jupiter_quote_cache: Arc::new(trigger_tcs::JupiterQuoteCache::default()), + now_ts, + }; + + let mut found_counter = 0; + + // Find interesting (pubkey, tcsid, volume) + for pubkey in accounts.iter() { + if let Some(error_entry) = self.errors.read().unwrap().had_too_many_errors( + LiqErrorType::TcsCollectionHard, + pubkey, + now, + ) { + trace!( + %pubkey, + error_entry.count, + "skip checking account for tcs, had errors recently", + ); + continue; + } + + let candidates = tcs_context.find_interesting_tcs_for_account(pubkey); + let mut error_guard = self.errors.write().unwrap(); + + match candidates { + Ok(v) => { + error_guard.clear(LiqErrorType::TcsCollectionHard, pubkey); + if v.is_empty() { + error_guard.clear(LiqErrorType::TcsCollectionPartial, pubkey); + error_guard.clear(LiqErrorType::TcsExecution, pubkey); + } else if v.iter().all(|it| it.is_ok()) { + error_guard.clear(LiqErrorType::TcsCollectionPartial, pubkey); + } else { + for it in v.iter() { + if let Err(e) = it { + error_guard.record( + LiqErrorType::TcsCollectionPartial, + pubkey, + e.to_string(), + ); + } + } + } + for interesting_candidate_res in v.iter() { + if let Ok(interesting_candidate) = interesting_candidate_res { + action(*interesting_candidate).expect("failed to send TCS candidate"); + found_counter += 1; + } + } + } + Err(e) => { + error_guard.record(LiqErrorType::TcsCollectionHard, pubkey, e.to_string()); + } + } + } + + return Ok(found_counter); + } + + pub async fn maybe_take_token_conditional_swap( + &mut self, + mut interesting_tcs: Vec<(Pubkey, u64, u64)>, + ) -> anyhow::Result { + let now_ts: u64 = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_secs(); + + let tcs_context = trigger_tcs::Context { + mango_client: self.mango_client.clone(), + account_fetcher: self.account_fetcher.clone(), + token_swap_info: self.token_swap_info.clone(), + config: self.trigger_tcs_config.clone(), + jupiter_quote_cache: Arc::new(trigger_tcs::JupiterQuoteCache::default()), + now_ts, + }; + + if interesting_tcs.is_empty() { + return Ok(false); + } + + let (txsigs, mut changed_pubkeys) = tcs_context + .execute_tcs(&mut interesting_tcs, self.errors.clone()) + .await?; + for pubkey in changed_pubkeys.iter() { + self.errors + .write() + .unwrap() + .clear(LiqErrorType::TcsExecution, pubkey); + } + if txsigs.is_empty() { + return Ok(false); + } + changed_pubkeys.push(self.mango_client.mango_account_address); + + // Force a refresh of affected accounts + let slot = self + .account_fetcher + .transaction_max_slot(&txsigs) + .await + .context("transaction_max_slot")?; + if let Err(e) = self + .account_fetcher + .refresh_accounts_via_rpc_until_slot( + &changed_pubkeys, + slot, + self.trigger_tcs_config.refresh_timeout, + ) + .await + { + info!(slot, "could not refresh after tcs execution: {}", e); + } + + Ok(true) + } +} diff --git a/bin/liquidator/src/token_swap_info.rs b/bin/liquidator/src/token_swap_info.rs index 8e4e018ad2..9f35e4f97d 100644 --- a/bin/liquidator/src/token_swap_info.rs +++ b/bin/liquidator/src/token_swap_info.rs @@ -6,7 +6,7 @@ use mango_v4_client::error_tracking::ErrorTracking; use tracing::*; use mango_v4::state::TokenIndex; -use mango_v4_client::jupiter; +use mango_v4_client::swap; use mango_v4_client::MangoClient; pub struct Config { @@ -15,7 +15,7 @@ pub struct Config { /// Size in quote_index-token native tokens to quote. pub quote_amount: u64, - pub jupiter_version: jupiter::Version, + pub jupiter_version: swap::Version, } #[derive(Clone)] @@ -84,7 +84,7 @@ impl TokenSwapInfoUpdater { lock.swap_infos.get(&token_index).cloned() } - fn in_per_out_price(route: &jupiter::Quote) -> f64 { + fn in_per_out_price(route: &swap::Quote) -> f64 { let in_amount = route.in_amount as f64; let out_amount = route.out_amount as f64; in_amount / out_amount @@ -149,7 +149,7 @@ impl TokenSwapInfoUpdater { let token_amount = (self.config.quote_amount as f64 * token_per_quote_oracle) as u64; let sell_route = self .mango_client - .jupiter() + .swap() .quote( token_mint, quote_mint, @@ -161,7 +161,7 @@ impl TokenSwapInfoUpdater { .await?; let buy_route = self .mango_client - .jupiter() + .swap() .quote( quote_mint, token_mint, diff --git a/bin/liquidator/src/trigger_tcs.rs b/bin/liquidator/src/trigger_tcs.rs index d421048460..0f902e149c 100644 --- a/bin/liquidator/src/trigger_tcs.rs +++ b/bin/liquidator/src/trigger_tcs.rs @@ -1,4 +1,5 @@ use std::collections::HashSet; +use std::time::Duration; use std::{ collections::HashMap, pin::Pin, @@ -12,9 +13,10 @@ use mango_v4::{ i80f48::ClampToInt, state::{Bank, MangoAccountValue, TokenConditionalSwap, TokenIndex}, }; -use mango_v4_client::{chain_data, jupiter, MangoClient, TransactionBuilder}; +use mango_v4_client::{chain_data, swap, MangoClient, TransactionBuilder}; use anyhow::Context as AnyhowContext; +use mango_v4::accounts_ix::HealthCheckKind::MaintRatio; use solana_sdk::signature::Signature; use tracing::*; use {fixed::types::I80F48, solana_sdk::pubkey::Pubkey}; @@ -55,6 +57,7 @@ pub enum Mode { #[derive(Clone)] pub struct Config { + pub refresh_timeout: Duration, pub min_health_ratio: f64, pub max_trigger_quote_amount: u64, pub compute_limit_for_trigger: u32, @@ -70,7 +73,7 @@ pub struct Config { /// Can be set to 0 to allow executions of any size. pub min_buy_fraction: f64, - pub jupiter_version: jupiter::Version, + pub jupiter_version: swap::Version, pub jupiter_slippage_bps: u64, pub mode: Mode, @@ -121,9 +124,9 @@ impl JupiterQuoteCache { output_mint: Pubkey, input_amount: u64, slippage_bps: u64, - version: jupiter::Version, + version: swap::Version, max_in_per_out_price: f64, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let cache_entry = self.cache_entry(input_mint, output_mint); let held_lock = { @@ -181,10 +184,10 @@ impl JupiterQuoteCache { output_mint: Pubkey, input_amount: u64, slippage_bps: u64, - version: jupiter::Version, - ) -> anyhow::Result<(f64, jupiter::Quote)> { + version: swap::Version, + ) -> anyhow::Result<(f64, swap::Quote)> { let quote = client - .jupiter() + .swap() .quote( input_mint, output_mint, @@ -205,8 +208,8 @@ impl JupiterQuoteCache { output_mint: Pubkey, input_amount: u64, slippage_bps: u64, - version: jupiter::Version, - ) -> anyhow::Result<(f64, jupiter::Quote)> { + version: swap::Version, + ) -> anyhow::Result<(f64, swap::Quote)> { match self .quote( client, @@ -252,11 +255,10 @@ impl JupiterQuoteCache { collateral_amount: u64, sell_amount: u64, slippage_bps: u64, - version: jupiter::Version, + version: swap::Version, max_sell_per_buy_price: f64, - ) -> anyhow::Result< - JupiterQuoteCacheResult<(f64, Option, Option)>, - > { + ) -> anyhow::Result, Option)>> + { // First check if we have cached prices for both legs and // if those break the specified limit let cached_collateral_to_buy = self.cached_price(collateral_mint, buy_mint).await; @@ -335,7 +337,7 @@ struct PreparedExecution { max_sell_token_to_liqor: u64, min_buy_token: u64, min_taker_price: f32, - jupiter_quote: Option, + jupiter_quote: Option, } struct PreparationResult { @@ -1000,7 +1002,7 @@ impl Context { pub async fn execute_tcs( &self, tcs: &mut [(Pubkey, u64, u64)], - error_tracking: &mut ErrorTracking, + error_tracking: Arc>>, ) -> anyhow::Result<(Vec, Vec)> { use rand::distributions::{Distribution, WeightedError, WeightedIndex}; @@ -1049,7 +1051,7 @@ impl Context { } Err(e) => { trace!(%result.pubkey, "preparation error {:?}", e); - error_tracking.record( + error_tracking.write().unwrap().record( LiqErrorType::TcsExecution, &result.pubkey, e.to_string(), @@ -1093,7 +1095,7 @@ impl Context { }; // start the new one - if let Some(job) = self.prepare_job(&pubkey, tcs_id, volume, error_tracking) { + if let Some(job) = self.prepare_job(&pubkey, tcs_id, volume, error_tracking.clone()) { pending_volume += volume; pending.push(job); } @@ -1130,7 +1132,11 @@ impl Context { Ok(v) => Some((pubkey, v)), Err(err) => { trace!(%pubkey, "execution error {:?}", err); - error_tracking.record(LiqErrorType::TcsExecution, &pubkey, err.to_string()); + error_tracking.write().unwrap().record( + LiqErrorType::TcsExecution, + &pubkey, + err.to_string(), + ); None } }); @@ -1145,12 +1151,14 @@ impl Context { pubkey: &Pubkey, tcs_id: u64, volume: u64, - error_tracking: &ErrorTracking, + error_tracking: Arc>>, ) -> Option + Send>>> { // Skip a pubkey if there've been too many errors recently - if let Some(error_entry) = - error_tracking.had_too_many_errors(LiqErrorType::TcsExecution, pubkey, Instant::now()) - { + if let Some(error_entry) = error_tracking.read().unwrap().had_too_many_errors( + LiqErrorType::TcsExecution, + pubkey, + Instant::now(), + ) { trace!( "skip checking for tcs on account {pubkey}, had {} errors recently", error_entry.count @@ -1191,7 +1199,7 @@ impl Context { // Jupiter quote is provided only for triggers, not close-expired let mut tx_builder = if let Some(jupiter_quote) = pending.jupiter_quote { self.mango_client - .jupiter() + .swap() .prepare_swap_transaction(&jupiter_quote) .await? } else { @@ -1225,6 +1233,27 @@ impl Context { .instructions .append(&mut trigger_ixs.instructions); + let (_, tcs) = liqee.token_conditional_swap_by_id(pending.tcs_id)?; + let affected_tokens = allowed_tokens + .iter() + .chain(&[tcs.buy_token_index, tcs.sell_token_index]) + .copied() + .collect_vec(); + let liqor = &self.mango_client.mango_account().await?; + tx_builder.instructions.append( + &mut self + .mango_client + .health_check_instruction( + liqor, + self.config.min_health_ratio, + affected_tokens, + vec![], + MaintRatio, + ) + .await? + .instructions, + ); + let txsig = tx_builder .send_and_confirm(&self.mango_client.client) .await?; diff --git a/bin/liquidator/src/tx_sender.rs b/bin/liquidator/src/tx_sender.rs new file mode 100644 index 0000000000..05027f6784 --- /dev/null +++ b/bin/liquidator/src/tx_sender.rs @@ -0,0 +1,241 @@ +use crate::liquidation_state::LiquidationState; +use crate::tcs_state::TcsState; +use crate::SharedState; +use anchor_lang::prelude::Pubkey; +use async_channel::{Receiver, Sender}; +use mango_v4_client::AsyncChannelSendUnlessFull; +use std::sync::{Arc, RwLock}; +use tokio::task::JoinHandle; +use tracing::{debug, error, trace}; + +enum WorkerTask { + Liquidation(Pubkey), + Tcs(Vec<(Pubkey, u64, u64)>), + + // Given two workers: #0=LIQ_only, #1=LIQ+TCS + // If they are both busy, and the scanning jobs find a new TCS and a new LIQ candidates and enqueue them in the channel + // Then if #1 wake up first, it will consume the LIQ candidate (LIQ always have priority) + // Then when #0 wake up, it will not find any LIQ candidate, and would not do anything (it won't take a TCS) + // But if we do nothing, #1 would never wake up again (no new task in channel) + // So we use this `GiveUpTcs` that will be handled by #0 by queuing a new signal the channel and will wake up #1 again + GiveUpTcs, + + // Can happen if TCS is batched (2 TCS enqueued, 2 workers waken, but first one take both tasks) + NoWork, +} + +pub fn spawn_tx_senders_job( + max_parallel_operations: u64, + enable_liquidation: bool, + tx_liq_trigger_receiver: Receiver<()>, + tx_tcs_trigger_receiver: Receiver<()>, + tx_tcs_trigger_sender: Sender<()>, + rebalance_trigger_sender: Sender<()>, + shared_state: Arc>, + liquidation: Box, + tcs: Box, +) -> Vec> { + if max_parallel_operations < 1 { + error!("max_parallel_operations must be >= 1"); + std::process::exit(1) + } + + let reserve_one_worker_for_liquidation = max_parallel_operations > 1 && enable_liquidation; + + let workers: Vec> = (0..max_parallel_operations) + .map(|worker_id| { + tokio::spawn({ + let shared_state = shared_state.clone(); + let receiver_liq = tx_liq_trigger_receiver.clone(); + let receiver_tcs = tx_tcs_trigger_receiver.clone(); + let sender_tcs = tx_tcs_trigger_sender.clone(); + let rebalance_trigger_sender = rebalance_trigger_sender.clone(); + let liquidation = liquidation.clone(); + let tcs = tcs.clone(); + async move { + worker_loop( + shared_state, + receiver_liq, + receiver_tcs, + sender_tcs, + rebalance_trigger_sender, + liquidation, + tcs, + worker_id, + reserve_one_worker_for_liquidation && worker_id == 0, + ) + .await; + } + }) + }) + .collect(); + + workers +} + +async fn worker_loop( + shared_state: Arc>, + liq_receiver: Receiver<()>, + tcs_receiver: Receiver<()>, + tcs_sender: Sender<()>, + rebalance_trigger_sender: Sender<()>, + mut liquidation: Box, + mut tcs: Box, + id: u64, + only_liquidation: bool, +) { + loop { + debug!( + "Worker #{} waiting for task (only_liq={})", + id, only_liquidation + ); + + let _ = if only_liquidation { + liq_receiver.recv().await.expect("receive failed") + } else { + tokio::select!( + _ = liq_receiver.recv() => {}, + _ = tcs_receiver.recv() => {}, + ) + }; + + // a task must be available to process + // find it in global shared state, and mark it as processing + let task = worker_pull_task(&shared_state, id, only_liquidation) + .expect("Worker woke up but has nothing to do"); + + // execute the task + let need_rebalancing = match &task { + WorkerTask::Liquidation(l) => worker_execute_liquidation(&mut liquidation, *l).await, + WorkerTask::Tcs(t) => worker_execute_tcs(&mut tcs, t.clone()).await, + WorkerTask::GiveUpTcs => worker_give_up_tcs(&tcs_sender).await, + WorkerTask::NoWork => false, + }; + + if need_rebalancing { + rebalance_trigger_sender.send_unless_full(()).unwrap(); + } + + // remove from shared state + worker_finalize_task(&shared_state, id, task, need_rebalancing); + } +} + +async fn worker_give_up_tcs(sender: &Sender<()>) -> bool { + sender.send(()).await.expect("sending task failed"); + false +} + +async fn worker_execute_tcs(tcs: &mut Box, candidates: Vec<(Pubkey, u64, u64)>) -> bool { + tcs.maybe_take_token_conditional_swap(candidates) + .await + .unwrap_or(false) +} + +async fn worker_execute_liquidation( + liquidation: &mut Box, + candidate: Pubkey, +) -> bool { + liquidation + .maybe_liquidate_and_log_error(&candidate) + .await + .unwrap_or(false) +} + +fn worker_pull_task( + shared_state: &Arc>, + id: u64, + only_liquidation: bool, +) -> anyhow::Result { + let mut writer = shared_state.write().unwrap(); + + // print out list of all task for debugging + for x in &writer.liquidation_candidates_accounts { + if !writer.processing_liquidation.contains(x) { + trace!(" - LIQ {:?}", x); + } + } + + // next liq task to execute + if let Some(liq_candidate) = writer + .liquidation_candidates_accounts + .iter() + .find(|x| !writer.processing_liquidation.contains(x)) + .copied() + { + debug!("worker #{} got a liq candidate -> {}", id, liq_candidate); + writer.processing_liquidation.insert(liq_candidate); + return Ok(WorkerTask::Liquidation(liq_candidate)); + } + + let tcs_todo = writer.interesting_tcs.len() - writer.processing_tcs.len(); + + if only_liquidation { + debug!("worker #{} giving up TCS (todo count: {})", id, tcs_todo); + return Ok(WorkerTask::GiveUpTcs); + } + + for x in &writer.interesting_tcs { + if !writer.processing_tcs.contains(x) { + trace!(" - TCS {:?}", x); + } + } + + // next tcs task to execute + let max_tcs_batch_size = 20; + let tcs_candidates: Vec<(Pubkey, u64, u64)> = writer + .interesting_tcs + .iter() + .filter(|x| !writer.processing_tcs.contains(x)) + .take(max_tcs_batch_size) + .copied() + .collect(); + + for tcs_candidate in &tcs_candidates { + debug!( + "worker #{} got a tcs candidate -> {:?} (out of {})", + id, + tcs_candidate, + writer.interesting_tcs.len() + ); + writer.processing_tcs.insert(tcs_candidate.clone()); + } + + if tcs_candidates.len() > 0 { + Ok(WorkerTask::Tcs(tcs_candidates)) + } else { + debug!("worker #{} got nothing", id); + Ok(WorkerTask::NoWork) + } +} + +fn worker_finalize_task( + shared_state: &Arc>, + id: u64, + task: WorkerTask, + done: bool, +) { + let mut writer = shared_state.write().unwrap(); + match task { + WorkerTask::Liquidation(liq) => { + debug!( + "worker #{} - checked liq {:?} with success ? {}", + id, liq, done + ); + writer.liquidation_candidates_accounts.shift_remove(&liq); + writer.processing_liquidation.remove(&liq); + } + WorkerTask::Tcs(tcs_list) => { + for tcs in tcs_list { + debug!( + "worker #{} - checked tcs {:?} with success ? {}", + id, tcs, done + ); + writer.interesting_tcs.shift_remove(&tcs); + writer.processing_tcs.remove(&tcs); + } + } + WorkerTask::GiveUpTcs => {} + WorkerTask::NoWork => {} + } +} diff --git a/bin/service-mango-health/src/main.rs b/bin/service-mango-health/src/main.rs index 9b3b5174a8..1baa76ed17 100644 --- a/bin/service-mango-health/src/main.rs +++ b/bin/service-mango-health/src/main.rs @@ -16,6 +16,11 @@ use crate::processors::health::HealthProcessor; use crate::processors::logger::LoggerProcessor; use crate::processors::persister::PersisterProcessor; +// jemalloc seems to be better at keeping the memory footprint reasonable over +// longer periods of time +#[global_allocator] +static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; + #[tokio::main] async fn main() -> anyhow::Result<()> { let args: Vec = std::env::args().collect(); @@ -64,7 +69,8 @@ async fn main() -> anyhow::Result<()> { ) .await?; - let mut jobs = vec![exit_processor.job, data_processor.job, health_processor.job]; + let mut jobs = vec![exit_processor.job, health_processor.job]; + jobs.extend(data_processor.jobs); if let Some(logger) = logger { jobs.push(logger.job) diff --git a/bin/service-mango-health/src/processors/data.rs b/bin/service-mango-health/src/processors/data.rs index 3a25923b90..feb24357f1 100644 --- a/bin/service-mango-health/src/processors/data.rs +++ b/bin/service-mango-health/src/processors/data.rs @@ -22,7 +22,7 @@ use tracing::warn; pub struct DataProcessor { pub channel: tokio::sync::broadcast::Sender, - pub job: JoinHandle<()>, + pub jobs: Vec>, pub chain_data: Arc>, } @@ -52,7 +52,7 @@ impl DataProcessor { ) -> anyhow::Result { let mut retry_counter = RetryCounter::new(2); let mango_group = Pubkey::from_str(&configuration.mango_group)?; - let mango_stream = + let (mango_stream, snapshot_job) = fail_or_retry!(retry_counter, Self::init_mango_source(configuration).await)?; let (sender, _) = tokio::sync::broadcast::channel(8192); let sender_clone = sender.clone(); @@ -98,7 +98,7 @@ impl DataProcessor { let result = DataProcessor { channel: sender, - job, + jobs: vec![job, snapshot_job], chain_data, }; @@ -147,7 +147,9 @@ impl DataProcessor { return Some(Other); } - async fn init_mango_source(configuration: &Configuration) -> anyhow::Result> { + async fn init_mango_source( + configuration: &Configuration, + ) -> anyhow::Result<(Receiver, JoinHandle<()>)> { // // Client setup // @@ -192,7 +194,7 @@ impl DataProcessor { // Getting solana account snapshots via jsonrpc // FUTURE: of what to fetch a snapshot - should probably take as an input - snapshot_source::start( + let snapshot_job = snapshot_source::start( snapshot_source::Config { rpc_http_url: configuration.rpc_http_url.clone(), mango_group, @@ -205,6 +207,6 @@ impl DataProcessor { account_update_sender, ); - Ok(account_update_receiver) + Ok((account_update_receiver, snapshot_job)) } } diff --git a/bin/settler/src/main.rs b/bin/settler/src/main.rs index 57f408a219..d1be966c40 100644 --- a/bin/settler/src/main.rs +++ b/bin/settler/src/main.rs @@ -178,7 +178,7 @@ async fn main() -> anyhow::Result<()> { // Getting solana account snapshots via jsonrpc // FUTURE: of what to fetch a snapshot - should probably take as an input - snapshot_source::start( + let snapshot_job = snapshot_source::start( snapshot_source::Config { rpc_http_url: rpc_url.clone(), mango_group, @@ -353,6 +353,7 @@ async fn main() -> anyhow::Result<()> { use futures::StreamExt; let mut jobs: futures::stream::FuturesUnordered<_> = vec![ + snapshot_job, data_job, settle_job, tcs_start_job, diff --git a/lib/client/Cargo.toml b/lib/client/Cargo.toml index cc1c29a9d7..b23dc9d15d 100644 --- a/lib/client/Cargo.toml +++ b/lib/client/Cargo.toml @@ -46,3 +46,4 @@ base64 = "0.13.0" bincode = "1.3.3" tracing = { version = "0.1", features = ["log"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] } +borsh = { version = "0.10.3", features = ["const-generics"] } diff --git a/lib/client/src/client.rs b/lib/client/src/client.rs index b6e3243a8d..598b53e208 100644 --- a/lib/client/src/client.rs +++ b/lib/client/src/client.rs @@ -17,7 +17,9 @@ use futures::{stream, StreamExt, TryFutureExt, TryStreamExt}; use itertools::Itertools; use tracing::*; -use mango_v4::accounts_ix::{Serum3OrderType, Serum3SelfTradeBehavior, Serum3Side}; +use mango_v4::accounts_ix::{ + HealthCheckKind, Serum3OrderType, Serum3SelfTradeBehavior, Serum3Side, +}; use mango_v4::accounts_zerocopy::KeyedAccountSharedData; use mango_v4::health::HealthCache; use mango_v4::state::{ @@ -25,14 +27,14 @@ use mango_v4::state::{ PlaceOrderType, SelfTradeBehavior, Serum3MarketIndex, Side, TokenIndex, INSURANCE_TOKEN_INDEX, }; -use crate::account_fetcher::*; use crate::confirm_transaction::{wait_for_transaction_confirmation, RpcConfirmTransactionConfig}; use crate::context::MangoGroupContext; use crate::gpa::{fetch_anchor_account, fetch_mango_accounts}; -use crate::health_cache; use crate::priority_fees::{FixedPriorityFeeProvider, PriorityFeeProvider}; +use crate::util; use crate::util::PreparedInstructions; -use crate::{jupiter, util}; +use crate::{account_fetcher::*, swap}; +use crate::{health_cache, Serum3MarketContext, TokenContext}; use solana_address_lookup_table_program::state::AddressLookupTable; use solana_client::nonblocking::rpc_client::RpcClient as RpcClientAsync; use solana_client::rpc_client::SerializableTransaction; @@ -80,6 +82,12 @@ pub struct ClientConfig { #[builder(default = "Duration::from_secs(60)")] pub timeout: Duration, + /// Jupiter Timeout, defaults to 30s + /// + /// This timeout applies to jupiter requests. + #[builder(default = "Duration::from_secs(30)")] + pub jupiter_timeout: Duration, + #[builder(default)] pub transaction_builder_config: TransactionBuilderConfig, @@ -97,6 +105,15 @@ pub struct ClientConfig { #[builder(default = "\"\".into()")] pub jupiter_token: String, + #[builder(default = "\"https://api.sanctum.so/v1\".into()")] + pub sanctum_url: String, + + /// Sanctum Timeout, defaults to 30s + /// + /// This timeout applies to jupiter requests. + #[builder(default = "Duration::from_secs(30)")] + pub sanctum_timeout: Duration, + /// Determines how fallback oracle accounts are provided to instructions. Defaults to Dynamic. #[builder(default = "FallbackOracleConfig::Dynamic")] pub fallback_oracle_config: FallbackOracleConfig, @@ -560,6 +577,76 @@ impl MangoClient { self.send_and_confirm_owner_tx(ixs.to_instructions()).await } + /// Assert that health of account is > N + pub async fn health_check_instruction( + &self, + account: &MangoAccountValue, + min_health_value: f64, + affected_tokens: Vec, + affected_perp_markets: Vec, + check_kind: HealthCheckKind, + ) -> anyhow::Result { + let (health_check_metas, health_cu) = self + .derive_health_check_remaining_account_metas( + account, + affected_tokens, + vec![], + affected_perp_markets, + ) + .await?; + + let ixs = PreparedInstructions::from_vec( + vec![Instruction { + program_id: mango_v4::id(), + accounts: { + let mut ams = anchor_lang::ToAccountMetas::to_account_metas( + &mango_v4::accounts::HealthCheck { + group: self.group(), + account: self.mango_account_address, + }, + None, + ); + ams.extend(health_check_metas.into_iter()); + ams + }, + data: anchor_lang::InstructionData::data(&mango_v4::instruction::HealthCheck { + min_health_value, + check_kind, + }), + }], + self.instruction_cu(health_cu), + ); + Ok(ixs) + } + + /// Avoid executing same instruction multiple time + pub async fn sequence_check_instruction( + &self, + mango_account_address: &Pubkey, + mango_account: &MangoAccountValue, + ) -> anyhow::Result { + let ixs = PreparedInstructions::from_vec( + vec![Instruction { + program_id: mango_v4::id(), + accounts: { + anchor_lang::ToAccountMetas::to_account_metas( + &mango_v4::accounts::SequenceCheck { + group: self.group(), + account: *mango_account_address, + owner: mango_account.fixed.owner, + }, + None, + ) + }, + data: anchor_lang::InstructionData::data(&mango_v4::instruction::SequenceCheck { + expected_sequence_number: mango_account.fixed.sequence_number, + }), + }], + self.context.compute_estimates.cu_for_sequence_check, + ); + Ok(ixs) + } + /// Creates token withdraw instructions for the MangoClient's account/owner. /// The `account` state is passed in separately so changes during the tx can be /// accounted for when deriving health accounts. @@ -613,7 +700,7 @@ impl MangoClient { }), }, ], - self.instruction_cu(health_cu), + self.instruction_cu(health_cu) + self.context.compute_estimates.cu_per_associated_token_account_creation, ); Ok(ixs) } @@ -1084,6 +1171,17 @@ impl MangoClient { let account = self.mango_account().await?; let open_orders = account.serum3_orders(market_index).unwrap().open_orders; + let ix = self.serum3_settle_funds_instruction(s3, base, quote, open_orders); + self.send_and_confirm_owner_tx(ix.to_instructions()).await + } + + pub fn serum3_settle_funds_instruction( + &self, + s3: &Serum3MarketContext, + base: &TokenContext, + quote: &TokenContext, + open_orders: Pubkey, + ) -> PreparedInstructions { let ix = Instruction { program_id: mango_v4::id(), accounts: anchor_lang::ToAccountMetas::to_account_metas( @@ -1116,7 +1214,11 @@ impl MangoClient { fees_to_dao: true, }), }; - self.send_and_confirm_owner_tx(vec![ix]).await + + PreparedInstructions::from_single( + ix, + self.context.compute_estimates.cu_per_mango_instruction, + ) } pub fn serum3_cancel_all_orders_instruction( @@ -2091,14 +2193,71 @@ impl MangoClient { )) } - // jupiter + // Swap (jupiter, sanctum) + pub fn swap(&self) -> swap::Swap { + swap::Swap { mango_client: self } + } + + pub fn jupiter_v6(&self) -> swap::jupiter_v6::JupiterV6 { + swap::jupiter_v6::JupiterV6 { + mango_client: self, + timeout_duration: self.client.config.jupiter_timeout, + } + } - pub fn jupiter_v6(&self) -> jupiter::v6::JupiterV6 { - jupiter::v6::JupiterV6 { mango_client: self } + pub fn sanctum(&self) -> swap::sanctum::Sanctum { + swap::sanctum::Sanctum { + mango_client: self, + timeout_duration: self.client.config.sanctum_timeout, + } } - pub fn jupiter(&self) -> jupiter::Jupiter { - jupiter::Jupiter { mango_client: self } + pub(crate) async fn deserialize_instructions_and_alts( + &self, + message: &solana_sdk::message::VersionedMessage, + ) -> anyhow::Result<(Vec, Vec)> { + let lookups = message.address_table_lookups().unwrap_or_default(); + let address_lookup_tables = self + .fetch_address_lookup_tables(lookups.iter().map(|a| &a.account_key)) + .await?; + + let mut account_keys = message.static_account_keys().to_vec(); + for (lookups, table) in lookups.iter().zip(address_lookup_tables.iter()) { + account_keys.extend( + lookups + .writable_indexes + .iter() + .map(|&index| table.addresses[index as usize]), + ); + } + for (lookups, table) in lookups.iter().zip(address_lookup_tables.iter()) { + account_keys.extend( + lookups + .readonly_indexes + .iter() + .map(|&index| table.addresses[index as usize]), + ); + } + + let compiled_ix = message + .instructions() + .iter() + .map(|ci| solana_sdk::instruction::Instruction { + program_id: *ci.program_id(&account_keys), + accounts: ci + .accounts + .iter() + .map(|&index| AccountMeta { + pubkey: account_keys[index as usize], + is_signer: message.is_signer(index.into()), + is_writable: message.is_maybe_writable(index.into()), + }) + .collect(), + data: ci.data.clone(), + }) + .collect(); + + Ok((compiled_ix, address_lookup_tables)) } pub async fn fetch_address_lookup_table( @@ -2423,6 +2582,11 @@ impl TransactionBuilder { length: bytes.len(), }) } + + pub fn append(&mut self, prepared_instructions: PreparedInstructions) { + self.instructions + .extend(prepared_instructions.to_instructions()); + } } /// Do some manual unpacking on some ClientErrors diff --git a/lib/client/src/context.rs b/lib/client/src/context.rs index 185555a043..cc9ca95946 100644 --- a/lib/client/src/context.rs +++ b/lib/client/src/context.rs @@ -122,6 +122,8 @@ pub struct ComputeEstimates { pub cu_per_oracle_fallback: u32, pub cu_per_charge_collateral_fees: u32, pub cu_per_charge_collateral_fees_token: u32, + pub cu_for_sequence_check: u32, + pub cu_per_associated_token_account_creation: u32, } impl Default for ComputeEstimates { @@ -145,6 +147,9 @@ impl Default for ComputeEstimates { cu_per_charge_collateral_fees: 20_000, // per-chargable-token cost cu_per_charge_collateral_fees_token: 15_000, + // measured around 8k, see test_basics + cu_for_sequence_check: 10_000, + cu_per_associated_token_account_creation: 21_000, } } } diff --git a/lib/client/src/gpa.rs b/lib/client/src/gpa.rs index e96aa54181..7c02ed8c02 100644 --- a/lib/client/src/gpa.rs +++ b/lib/client/src/gpa.rs @@ -1,11 +1,12 @@ use anchor_lang::{AccountDeserialize, Discriminator}; +use futures::{stream, StreamExt}; use mango_v4::state::{Bank, MangoAccount, MangoAccountValue, MintInfo, PerpMarket, Serum3Market}; use solana_account_decoder::UiAccountEncoding; use solana_client::nonblocking::rpc_client::RpcClient as RpcClientAsync; use solana_client::rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig}; use solana_client::rpc_filter::{Memcmp, RpcFilterType}; -use solana_sdk::account::AccountSharedData; +use solana_sdk::account::{Account, AccountSharedData}; use solana_sdk::pubkey::Pubkey; pub async fn fetch_mango_accounts( @@ -148,3 +149,49 @@ pub async fn fetch_multiple_accounts( .map(|(acc, key)| (*key, acc.unwrap().into())) .collect()) } + +/// Fetch multiple account using one request per chunk of `max_chunk_size` accounts +/// Can execute in parallel up to `parallel_rpc_requests` +/// +/// WARNING: some accounts requested may be missing from the result +pub async fn fetch_multiple_accounts_in_chunks( + rpc: &RpcClientAsync, + keys: &[Pubkey], + max_chunk_size: usize, + parallel_rpc_requests: usize, +) -> anyhow::Result> { + let config = RpcAccountInfoConfig { + encoding: Some(UiAccountEncoding::Base64), + ..RpcAccountInfoConfig::default() + }; + + let raw_results = stream::iter(keys) + .chunks(max_chunk_size) + .map(|keys| { + let account_info_config = config.clone(); + async move { + let keys = keys.iter().map(|x| **x).collect::>(); + let req_res = rpc + .get_multiple_accounts_with_config(&keys, account_info_config) + .await; + + match req_res { + Ok(v) => Ok(keys.into_iter().zip(v.value).collect::>()), + Err(e) => Err(e), + } + } + }) + .buffer_unordered(parallel_rpc_requests) + .collect::>() + .await; + + let result = raw_results + .into_iter() + .collect::, _>>()? + .into_iter() + .flatten() + .filter_map(|(pubkey, account_opt)| account_opt.map(|acc| (pubkey, acc))) + .collect::>(); + + Ok(result) +} diff --git a/lib/client/src/lib.rs b/lib/client/src/lib.rs index 882a931f68..620477f2a5 100644 --- a/lib/client/src/lib.rs +++ b/lib/client/src/lib.rs @@ -13,11 +13,11 @@ mod context; pub mod error_tracking; pub mod gpa; pub mod health_cache; -pub mod jupiter; pub mod perp_pnl; pub mod priority_fees; pub mod priority_fees_cli; pub mod snapshot_source; +pub mod swap; mod util; pub mod websocket_source; diff --git a/lib/client/src/snapshot_source.rs b/lib/client/src/snapshot_source.rs index 44da48303d..d35ca54a95 100644 --- a/lib/client/src/snapshot_source.rs +++ b/lib/client/src/snapshot_source.rs @@ -16,6 +16,7 @@ use solana_rpc::rpc::rpc_accounts::AccountsDataClient; use solana_rpc::rpc::rpc_accounts_scan::AccountsScanClient; use std::str::FromStr; use std::time::{Duration, Instant}; +use tokio::task::JoinHandle; use tokio::time; use tracing::*; @@ -223,11 +224,15 @@ async fn feed_snapshots( Ok(()) } -pub fn start(config: Config, mango_oracles: Vec, sender: async_channel::Sender) { +pub fn start( + config: Config, + mango_oracles: Vec, + sender: async_channel::Sender, +) -> JoinHandle<()> { let mut poll_wait_first_snapshot = crate::delay_interval(time::Duration::from_secs(2)); let mut interval_between_snapshots = crate::delay_interval(config.snapshot_interval); - tokio::spawn(async move { + let snapshot_job = tokio::spawn(async move { let rpc_client = http::connect_with_options::(&config.rpc_http_url, true) .await .expect("always Ok"); @@ -260,4 +265,6 @@ pub fn start(config: Config, mango_oracles: Vec, sender: async_channel:: }; } }); + + snapshot_job } diff --git a/lib/client/src/jupiter/v6.rs b/lib/client/src/swap/jupiter_v6.rs similarity index 98% rename from lib/client/src/jupiter/v6.rs rename to lib/client/src/swap/jupiter_v6.rs index 6c73fc7417..91ef1dee1f 100644 --- a/lib/client/src/jupiter/v6.rs +++ b/lib/client/src/swap/jupiter_v6.rs @@ -1,4 +1,5 @@ use std::str::FromStr; +use std::time::Duration; use anchor_lang::prelude::Pubkey; use serde::{Deserialize, Serialize}; @@ -72,13 +73,7 @@ pub struct SwapRequest { #[derive(Deserialize, Serialize, Debug, Clone)] #[serde(rename_all = "camelCase")] -pub struct SwapResponse { - pub swap_transaction: String, -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct SwapInstructionsResponse { +pub struct JupiterSwapInstructionsResponse { pub token_ledger_instruction: Option, pub compute_budget_instructions: Option>, pub setup_instructions: Option>, @@ -139,6 +134,7 @@ impl TryFrom<&AccountMeta> for solana_sdk::instruction::AccountMeta { pub struct JupiterV6<'a> { pub mango_client: &'a MangoClient, + pub timeout_duration: Duration, } impl<'a> JupiterV6<'a> { @@ -204,6 +200,7 @@ impl<'a> JupiterV6<'a> { .http_client .get(format!("{}/quote", config.jupiter_v6_url)) .query(&query_args) + .timeout(self.timeout_duration) .send() .await .context("quote request to jupiter")?; @@ -290,11 +287,12 @@ impl<'a> JupiterV6<'a> { destination_token_account: None, // default to user ata quote_response: quote.clone(), }) + .timeout(self.timeout_duration) .send() .await .context("swap transaction request to jupiter")?; - let swap: SwapInstructionsResponse = util::http_error_handling(swap_response) + let swap: JupiterSwapInstructionsResponse = util::http_error_handling(swap_response) .await .context("error requesting jupiter swap")?; diff --git a/lib/client/src/jupiter/mod.rs b/lib/client/src/swap/mod.rs similarity index 69% rename from lib/client/src/jupiter/mod.rs rename to lib/client/src/swap/mod.rs index e8eeeb2ed2..4a0c9c9d02 100644 --- a/lib/client/src/jupiter/mod.rs +++ b/lib/client/src/swap/mod.rs @@ -1,4 +1,6 @@ -pub mod v6; +pub mod jupiter_v6; +pub mod sanctum; +pub mod sanctum_state; use anchor_lang::prelude::*; use std::str::FromStr; @@ -10,13 +12,15 @@ use fixed::types::I80F48; pub enum Version { Mock, V6, + Sanctum, } #[derive(Clone)] #[allow(clippy::large_enum_variant)] pub enum RawQuote { Mock, - V6(v6::QuoteResponse), + V6(jupiter_v6::QuoteResponse), + Sanctum(sanctum::QuoteResponse), } #[derive(Clone)] @@ -30,7 +34,7 @@ pub struct Quote { } impl Quote { - pub fn try_from_v6(query: v6::QuoteResponse) -> anyhow::Result { + pub fn try_from_v6(query: jupiter_v6::QuoteResponse) -> anyhow::Result { Ok(Quote { input_mint: Pubkey::from_str(&query.input_mint)?, output_mint: Pubkey::from_str(&query.output_mint)?, @@ -45,6 +49,25 @@ impl Quote { }) } + pub fn try_from_sanctum( + input_mint: Pubkey, + output_mint: Pubkey, + query: sanctum::QuoteResponse, + ) -> anyhow::Result { + Ok(Quote { + input_mint: input_mint, + output_mint: output_mint, + price_impact_pct: query.fee_pct.parse()?, + in_amount: query + .in_amount + .as_ref() + .map(|a| a.parse()) + .unwrap_or(Ok(0))?, + out_amount: query.out_amount.parse()?, + raw: RawQuote::Sanctum(query), + }) + } + pub fn first_route_label(&self) -> String { let label_maybe = match &self.raw { RawQuote::Mock => Some("mock".into()), @@ -54,16 +77,17 @@ impl Quote { .and_then(|v| v.swap_info.as_ref()) .and_then(|v| v.label.as_ref()) .cloned(), + RawQuote::Sanctum(raw) => Some(raw.swap_src.clone()), }; label_maybe.unwrap_or_else(|| "unknown".into()) } } -pub struct Jupiter<'a> { +pub struct Swap<'a> { pub mango_client: &'a MangoClient, } -impl<'a> Jupiter<'a> { +impl<'a> Swap<'a> { async fn quote_mock( &self, input_mint: Pubkey, @@ -123,6 +147,14 @@ impl<'a> Jupiter<'a> { ) .await?, )?, + Version::Sanctum => Quote::try_from_sanctum( + input_mint, + output_mint, + self.mango_client + .sanctum() + .quote(input_mint, output_mint, amount) + .await?, + )?, }) } @@ -138,6 +170,18 @@ impl<'a> Jupiter<'a> { .prepare_swap_transaction(raw) .await } + RawQuote::Sanctum(raw) => { + let max_slippage_bps = (quote.price_impact_pct * 100.0).ceil() as u64; + self.mango_client + .sanctum() + .prepare_swap_transaction( + quote.input_mint, + quote.output_mint, + max_slippage_bps, + raw, + ) + .await + } } } } diff --git a/lib/client/src/swap/sanctum.rs b/lib/client/src/swap/sanctum.rs new file mode 100644 index 0000000000..ff48b5c488 --- /dev/null +++ b/lib/client/src/swap/sanctum.rs @@ -0,0 +1,401 @@ +use std::collections::HashSet; +use std::str::FromStr; + +use anchor_lang::{system_program, Id}; +use anchor_spl::token::Token; +use anyhow::Context; +use bincode::Options; +use mango_v4::accounts_zerocopy::AccountReader; +use serde::{Deserialize, Serialize}; +use solana_address_lookup_table_program::state::AddressLookupTable; +use solana_client::nonblocking::rpc_client::RpcClient; +use solana_sdk::account::Account; +use solana_sdk::{instruction::Instruction, pubkey::Pubkey, signature::Signature}; +use std::time::Duration; + +use crate::gpa::fetch_multiple_accounts_in_chunks; +use crate::swap::sanctum_state; +use crate::{util, MangoClient, TransactionBuilder}; +use borsh::BorshDeserialize; + +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct QuoteResponse { + pub in_amount: Option, + pub out_amount: String, + pub fee_amount: String, + pub fee_mint: String, + pub fee_pct: String, + pub swap_src: String, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct SwapRequest { + pub amount: String, + pub quoted_amount: String, + pub input: String, + pub mode: String, + pub output_lst_mint: String, + pub signer: String, + pub swap_src: String, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct SanctumSwapResponse { + pub tx: String, +} + +pub struct Sanctum<'a> { + pub mango_client: &'a MangoClient, + pub timeout_duration: Duration, +} + +impl<'a> Sanctum<'a> { + pub async fn quote( + &self, + input_mint: Pubkey, + output_mint: Pubkey, + amount: u64, + ) -> anyhow::Result { + if input_mint == output_mint { + anyhow::bail!("Need two distinct mint to swap"); + } + + let mut account = self.mango_client.mango_account().await?; + let input_token_index = self + .mango_client + .context + .token_by_mint(&input_mint)? + .token_index; + let output_token_index = self + .mango_client + .context + .token_by_mint(&output_mint)? + .token_index; + account.ensure_token_position(input_token_index)?; + account.ensure_token_position(output_token_index)?; + + let query_args = vec![ + ("input", input_mint.to_string()), + ("outputLstMint", output_mint.to_string()), + ("amount", format!("{}", amount)), + ]; + let config = self.mango_client.client.config(); + + let response = self + .mango_client + .http_client + .get(format!("{}/swap/quote", config.sanctum_url)) + .query(&query_args) + .timeout(self.timeout_duration) + .send() + .await + .context("quote request to sanctum")?; + let quote: QuoteResponse = + util::http_error_handling(response).await.with_context(|| { + format!("error requesting sanctum route between {input_mint} and {output_mint} (using url: {})", config.sanctum_url) + })?; + + Ok(quote) + } + + /// Find the instructions and account lookup tables for a sanctum swap through mango + pub async fn prepare_swap_transaction( + &self, + input_mint: Pubkey, + output_mint: Pubkey, + max_slippage_bps: u64, + quote: &QuoteResponse, + ) -> anyhow::Result { + tracing::info!("swapping using sanctum"); + + let source_token = self.mango_client.context.token_by_mint(&input_mint)?; + let target_token = self.mango_client.context.token_by_mint(&output_mint)?; + + let bank_ams = [source_token.first_bank(), target_token.first_bank()] + .into_iter() + .map(util::to_writable_account_meta) + .collect::>(); + + let vault_ams = [source_token.first_vault(), target_token.first_vault()] + .into_iter() + .map(util::to_writable_account_meta) + .collect::>(); + + let owner = self.mango_client.owner(); + let account = &self.mango_client.mango_account().await?; + + let token_ams = [source_token.mint, target_token.mint] + .into_iter() + .map(|mint| { + util::to_writable_account_meta( + anchor_spl::associated_token::get_associated_token_address(&owner, &mint), + ) + }) + .collect::>(); + + let source_loan = quote + .in_amount + .as_ref() + .map(|v| u64::from_str(v).unwrap()) + .unwrap_or(0); + let loan_amounts = vec![source_loan, 0u64]; + let num_loans: u8 = loan_amounts.len().try_into().unwrap(); + + // This relies on the fact that health account banks will be identical to the first_bank above! + let (health_ams, _health_cu) = self + .mango_client + .derive_health_check_remaining_account_metas( + account, + vec![source_token.token_index, target_token.token_index], + vec![source_token.token_index, target_token.token_index], + vec![], + ) + .await + .context("building health accounts")?; + + let config = self.mango_client.client.config(); + + let in_amount = quote + .in_amount + .clone() + .expect("sanctum require a in amount"); + let quote_amount_u64 = quote.out_amount.parse::()?; + let out_amount = ((quote_amount_u64 as f64) * (1.0 - (max_slippage_bps as f64) / 10_000.0)) + .ceil() as u64; + + let swap_response = self + .mango_client + .http_client + .post(format!("{}/swap", config.sanctum_url)) + .json(&SwapRequest { + amount: in_amount.clone(), + quoted_amount: out_amount.to_string(), + input: input_mint.to_string(), + mode: "ExactIn".to_string(), + output_lst_mint: output_mint.to_string(), + signer: owner.to_string(), + swap_src: quote.swap_src.clone(), + }) + .timeout(self.timeout_duration) + .send() + .await + .context("swap transaction request to sanctum")?; + + let swap_r: SanctumSwapResponse = util::http_error_handling(swap_response) + .await + .context("error requesting sanctum swap")?; + + let tx = bincode::options() + .with_fixint_encoding() + .reject_trailing_bytes() + .deserialize::( + &base64::decode(&swap_r.tx).context("base64 decoding sanctum transaction")?, + ) + .context("parsing sanctum transaction")?; + + let (sanctum_ixs_orig, sanctum_alts) = self + .mango_client + .deserialize_instructions_and_alts(&tx.message) + .await?; + + let system_program = system_program::ID; + let ata_program = anchor_spl::associated_token::ID; + let token_program = anchor_spl::token::ID; + let compute_budget_program: Pubkey = solana_sdk::compute_budget::ID; + // these setup instructions should be placed outside of flashloan begin-end + let is_setup_ix = |k: Pubkey| -> bool { + k == ata_program || k == token_program || k == compute_budget_program + }; + let sync_native_pack = + anchor_spl::token::spl_token::instruction::TokenInstruction::SyncNative.pack(); + + // Remove auto wrapping of SOL->wSOL + let sanctum_ixs: Vec = sanctum_ixs_orig + .clone() + .into_iter() + .filter(|ix| { + !(ix.program_id == system_program) + && !(ix.program_id == token_program && ix.data == sync_native_pack) + }) + .collect(); + + let sanctum_action_ix_begin = sanctum_ixs + .iter() + .position(|ix| !is_setup_ix(ix.program_id)) + .ok_or_else(|| { + anyhow::anyhow!("sanctum swap response only had setup-like instructions") + })?; + let sanctum_action_ix_end = sanctum_ixs.len() + - sanctum_ixs + .iter() + .rev() + .position(|ix| !is_setup_ix(ix.program_id)) + .unwrap(); + + let mut instructions: Vec = Vec::new(); + + for ix in &sanctum_ixs[..sanctum_action_ix_begin] { + instructions.push(ix.clone()); + } + + // Ensure the source token account is created (sanctum takes care of the output account) + instructions.push( + spl_associated_token_account::instruction::create_associated_token_account_idempotent( + &owner, + &owner, + &source_token.mint, + &Token::id(), + ), + ); + + instructions.push(Instruction { + program_id: mango_v4::id(), + accounts: { + let mut ams = anchor_lang::ToAccountMetas::to_account_metas( + &mango_v4::accounts::FlashLoanBegin { + account: self.mango_client.mango_account_address, + owner, + token_program: Token::id(), + instructions: solana_sdk::sysvar::instructions::id(), + }, + None, + ); + ams.extend(bank_ams); + ams.extend(vault_ams.clone()); + ams.extend(token_ams.clone()); + ams.push(util::to_readonly_account_meta(self.mango_client.group())); + ams + }, + data: anchor_lang::InstructionData::data(&mango_v4::instruction::FlashLoanBegin { + loan_amounts, + }), + }); + + for ix in &sanctum_ixs[sanctum_action_ix_begin..sanctum_action_ix_end] { + instructions.push(ix.clone()); + } + + instructions.push(Instruction { + program_id: mango_v4::id(), + accounts: { + let mut ams = anchor_lang::ToAccountMetas::to_account_metas( + &mango_v4::accounts::FlashLoanEnd { + account: self.mango_client.mango_account_address, + owner, + token_program: Token::id(), + }, + None, + ); + ams.extend(health_ams); + ams.extend(vault_ams); + ams.extend(token_ams); + ams.push(util::to_readonly_account_meta(self.mango_client.group())); + ams + }, + data: anchor_lang::InstructionData::data(&mango_v4::instruction::FlashLoanEndV2 { + num_loans, + flash_loan_type: mango_v4::accounts_ix::FlashLoanType::Swap, + }), + }); + + for ix in &sanctum_ixs[sanctum_action_ix_end..] { + instructions.push(ix.clone()); + } + + let mut address_lookup_tables = self.mango_client.mango_address_lookup_tables().await?; + address_lookup_tables.extend(sanctum_alts.into_iter()); + + let payer = owner; // maybe use fee_payer? but usually it's the same + + Ok(TransactionBuilder { + instructions, + address_lookup_tables, + payer, + signers: vec![self.mango_client.owner.clone()], + config: self + .mango_client + .client + .config() + .transaction_builder_config + .clone(), + }) + } + + pub async fn swap( + &self, + input_mint: Pubkey, + output_mint: Pubkey, + max_slippage_bps: u64, + amount: u64, + ) -> anyhow::Result { + let route = self.quote(input_mint, output_mint, amount).await?; + + let tx_builder = self + .prepare_swap_transaction(input_mint, output_mint, max_slippage_bps, &route) + .await?; + + tx_builder.send_and_confirm(&self.mango_client.client).await + } +} + +pub async fn load_supported_token_mints( + live_rpc_client: &RpcClient, +) -> anyhow::Result> { + let address = Pubkey::from_str("EhWxBHdmQ3yDmPzhJbKtGMM9oaZD42emt71kSieghy5")?; + + let lookup_table_data = live_rpc_client.get_account(&address).await?; + let lookup_table = AddressLookupTable::deserialize(&lookup_table_data.data())?; + let accounts: Vec = + fetch_multiple_accounts_in_chunks(live_rpc_client, &lookup_table.addresses, 100, 1) + .await? + .into_iter() + .map(|x| x.1) + .collect(); + + let mut lst_mints = HashSet::new(); + for account in accounts { + let account = Account::from(account); + let mut account_data = account.data(); + let t = sanctum_state::StakePool::deserialize(&mut account_data); + if let Ok(d) = t { + lst_mints.insert(d.pool_mint); + } + } + + // Hardcoded for now + lst_mints.insert( + Pubkey::from_str("CgntPoLka5pD5fesJYhGmUCF8KU1QS1ZmZiuAuMZr2az").expect("invalid lst mint"), + ); + lst_mints.insert( + Pubkey::from_str("7ge2xKsZXmqPxa3YmXxXmzCp9Hc2ezrTxh6PECaxCwrL").expect("invalid lst mint"), + ); + lst_mints.insert( + Pubkey::from_str("GUAMR8ciiaijraJeLDEDrFVaueLm9YzWWY9R7CBPL9rA").expect("invalid lst mint"), + ); + lst_mints.insert( + Pubkey::from_str("Jito4APyf642JPZPx3hGc6WWJ8zPKtRbRs4P815Awbb").expect("invalid lst mint"), + ); + lst_mints.insert( + Pubkey::from_str("CtMyWsrUtAwXWiGr9WjHT5fC3p3fgV8cyGpLTo2LJzG1").expect("invalid lst mint"), + ); + lst_mints.insert( + Pubkey::from_str("2qyEeSAWKfU18AFthrF7JA8z8ZCi1yt76Tqs917vwQTV").expect("invalid lst mint"), + ); + lst_mints.insert( + Pubkey::from_str("DqhH94PjkZsjAqEze2BEkWhFQJ6EyU6MdtMphMgnXqeK").expect("invalid lst mint"), + ); + lst_mints.insert( + Pubkey::from_str("F8h46pYkaqPJNP2MRkUUUtRkf8efCkpoqehn9g1bTTm7").expect("invalid lst mint"), + ); + lst_mints.insert( + Pubkey::from_str("5oc4nmbNTda9fx8Tw57ShLD132aqDK65vuHH4RU1K4LZ").expect("invalid lst mint"), + ); + lst_mints.insert( + Pubkey::from_str("stk9ApL5HeVAwPLr3TLhDXdZS8ptVu7zp6ov8HFDuMi").expect("invalid lst mint"), + ); + + Ok(lst_mints) +} diff --git a/lib/client/src/swap/sanctum_state.rs b/lib/client/src/swap/sanctum_state.rs new file mode 100644 index 0000000000..ea37e04832 --- /dev/null +++ b/lib/client/src/swap/sanctum_state.rs @@ -0,0 +1,158 @@ +use { + borsh::BorshDeserialize, + solana_sdk::{pubkey::Pubkey, stake::state::Lockup}, +}; + +#[derive(Clone, Debug, PartialEq, BorshDeserialize)] +pub enum AccountType { + /// If the account has not been initialized, the enum will be 0 + Uninitialized, + /// Stake pool + StakePool, + /// Validator stake list + ValidatorList, +} + +#[repr(C)] +#[derive(Clone, Debug, PartialEq, BorshDeserialize)] +pub struct StakePool { + /// Account type, must be StakePool currently + pub account_type: AccountType, + + /// Manager authority, allows for updating the staker, manager, and fee + /// account + pub manager: Pubkey, + + /// Staker authority, allows for adding and removing validators, and + /// managing stake distribution + pub staker: Pubkey, + + /// Stake deposit authority + /// + /// If a depositor pubkey is specified on initialization, then deposits must + /// be signed by this authority. If no deposit authority is specified, + /// then the stake pool will default to the result of: + /// `Pubkey::find_program_address( + /// &[&stake_pool_address.as_ref(), b"deposit"], + /// program_id, + /// )` + pub stake_deposit_authority: Pubkey, + + /// Stake withdrawal authority bump seed + /// for `create_program_address(&[state::StakePool account, "withdrawal"])` + pub stake_withdraw_bump_seed: u8, + + /// Validator stake list storage account + pub validator_list: Pubkey, + + /// Reserve stake account, holds deactivated stake + pub reserve_stake: Pubkey, + + /// Pool Mint + pub pool_mint: Pubkey, + + /// Manager fee account + pub manager_fee_account: Pubkey, + + /// Pool token program id + pub token_program_id: Pubkey, + + /// Total stake under management. + /// Note that if `last_update_epoch` does not match the current epoch then + /// this field may not be accurate + pub total_lamports: u64, + + /// Total supply of pool tokens (should always match the supply in the Pool + /// Mint) + pub pool_token_supply: u64, + + /// Last epoch the `total_lamports` field was updated + pub last_update_epoch: u64, + + /// Lockup that all stakes in the pool must have + pub lockup: Lockup, + + /// Fee taken as a proportion of rewards each epoch + pub epoch_fee: Fee, + + /// Fee for next epoch + pub next_epoch_fee: FutureEpoch, + + /// Preferred deposit validator vote account pubkey + pub preferred_deposit_validator_vote_address: Option, + + /// Preferred withdraw validator vote account pubkey + pub preferred_withdraw_validator_vote_address: Option, + + /// Fee assessed on stake deposits + pub stake_deposit_fee: Fee, + + /// Fee assessed on withdrawals + pub stake_withdrawal_fee: Fee, + + /// Future stake withdrawal fee, to be set for the following epoch + pub next_stake_withdrawal_fee: FutureEpoch, + + /// Fees paid out to referrers on referred stake deposits. + /// Expressed as a percentage (0 - 100) of deposit fees. + /// i.e. `stake_deposit_fee`% of stake deposited is collected as deposit + /// fees for every deposit and `stake_referral_fee`% of the collected + /// stake deposit fees is paid out to the referrer + pub stake_referral_fee: u8, + + /// Toggles whether the `DepositSol` instruction requires a signature from + /// this `sol_deposit_authority` + pub sol_deposit_authority: Option, + + /// Fee assessed on SOL deposits + pub sol_deposit_fee: Fee, + + /// Fees paid out to referrers on referred SOL deposits. + /// Expressed as a percentage (0 - 100) of SOL deposit fees. + /// i.e. `sol_deposit_fee`% of SOL deposited is collected as deposit fees + /// for every deposit and `sol_referral_fee`% of the collected SOL + /// deposit fees is paid out to the referrer + pub sol_referral_fee: u8, + + /// Toggles whether the `WithdrawSol` instruction requires a signature from + /// the `deposit_authority` + pub sol_withdraw_authority: Option, + + /// Fee assessed on SOL withdrawals + pub sol_withdrawal_fee: Fee, + + /// Future SOL withdrawal fee, to be set for the following epoch + pub next_sol_withdrawal_fee: FutureEpoch, + + /// Last epoch's total pool tokens, used only for APR estimation + pub last_epoch_pool_token_supply: u64, + + /// Last epoch's total lamports, used only for APR estimation + pub last_epoch_total_lamports: u64, +} + +/// Fee rate as a ratio, minted on `UpdateStakePoolBalance` as a proportion of +/// the rewards +/// If either the numerator or the denominator is 0, the fee is considered to be +/// 0 +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, BorshDeserialize)] +pub struct Fee { + /// denominator of the fee ratio + pub denominator: u64, + /// numerator of the fee ratio + pub numerator: u64, +} + +/// Wrapper type that "counts down" epochs, which is Borsh-compatible with the +/// native `Option` +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, BorshDeserialize)] +pub enum FutureEpoch { + /// Nothing is set + None, + /// Value is ready after the next epoch boundary + One(T), + /// Value is ready after two epoch boundaries + Two(T), +} diff --git a/lib/client/src/util.rs b/lib/client/src/util.rs index f54d6cac9f..cd562e33c8 100644 --- a/lib/client/src/util.rs +++ b/lib/client/src/util.rs @@ -20,19 +20,29 @@ impl AnyhowWrap for Result { /// Push to an async_channel::Sender and ignore if the channel is full pub trait AsyncChannelSendUnlessFull { /// Send a message if the channel isn't full - fn send_unless_full(&self, msg: T) -> Result<(), async_channel::SendError>; + fn send_unless_full(&self, msg: T) -> anyhow::Result<()>; } impl AsyncChannelSendUnlessFull for async_channel::Sender { - fn send_unless_full(&self, msg: T) -> Result<(), async_channel::SendError> { + fn send_unless_full(&self, msg: T) -> anyhow::Result<()> { use async_channel::*; match self.try_send(msg) { Ok(()) => Ok(()), - Err(TrySendError::Closed(msg)) => Err(async_channel::SendError(msg)), + Err(TrySendError::Closed(_)) => Err(anyhow::format_err!("channel is closed")), Err(TrySendError::Full(_)) => Ok(()), } } } +impl AsyncChannelSendUnlessFull for tokio::sync::mpsc::Sender { + fn send_unless_full(&self, msg: T) -> anyhow::Result<()> { + use tokio::sync::mpsc::*; + match self.try_send(msg) { + Ok(()) => Ok(()), + Err(error::TrySendError::Closed(_)) => Err(anyhow::format_err!("channel is closed")), + Err(error::TrySendError::Full(_)) => Ok(()), + } + } +} /// Like tokio::time::interval(), but with Delay as default MissedTickBehavior /// diff --git a/programs/mango-v4/src/accounts_ix/health_check.rs b/programs/mango-v4/src/accounts_ix/health_check.rs index c405a4fa30..677dd0d3d3 100644 --- a/programs/mango-v4/src/accounts_ix/health_check.rs +++ b/programs/mango-v4/src/accounts_ix/health_check.rs @@ -17,7 +17,7 @@ pub enum HealthCheckKind { #[derive(Accounts)] pub struct HealthCheck<'info> { #[account( - constraint = group.load()?.is_ix_enabled(IxGate::SequenceCheck) @ MangoError::IxIsDisabled, + constraint = group.load()?.is_ix_enabled(IxGate::HealthCheck) @ MangoError::IxIsDisabled, )] pub group: AccountLoader<'info, Group>, diff --git a/programs/mango-v4/src/accounts_ix/perp_edit_market.rs b/programs/mango-v4/src/accounts_ix/perp_edit_market.rs index f1d1937ca3..6716d4a6fe 100644 --- a/programs/mango-v4/src/accounts_ix/perp_edit_market.rs +++ b/programs/mango-v4/src/accounts_ix/perp_edit_market.rs @@ -17,4 +17,9 @@ pub struct PerpEditMarket<'info> { /// /// CHECK: The oracle can be one of several different account types pub oracle: UncheckedAccount<'info>, + + /// The fallback oracle account is optional and only used when set_fallback_oracle is true. + /// + /// CHECK: The fallback oracle can be one of several different account types + pub fallback_oracle: UncheckedAccount<'info>, } diff --git a/programs/mango-v4/src/health/account_retriever.rs b/programs/mango-v4/src/health/account_retriever.rs index 43bffd482e..76a9c46e12 100644 --- a/programs/mango-v4/src/health/account_retriever.rs +++ b/programs/mango-v4/src/health/account_retriever.rs @@ -26,6 +26,7 @@ use crate::state::{Bank, MangoAccountRef, PerpMarket, PerpMarketIndex, TokenInde /// are passed because health needs to be computed for different baskets in /// one instruction (such as for liquidation instructions). pub trait AccountRetriever { + /// Returns the token indexes of the available banks. Unordered and may have duplicates. fn available_banks(&self) -> Result>; fn bank_and_oracle( @@ -67,6 +68,9 @@ pub struct FixedOrderAccountRetriever { } /// Creates a FixedOrderAccountRetriever where all banks are present +/// +/// Note that this does not eagerly validate that the right accounts were passed. That +/// validation happens only when banks, perps etc are requested. pub fn new_fixed_order_account_retriever<'a, 'info>( ais: &'a [AccountInfo<'info>], account: &MangoAccountRef, @@ -84,6 +88,9 @@ pub fn new_fixed_order_account_retriever<'a, 'info>( /// A FixedOrderAccountRetriever with n_banks <= active_token_positions().count(), /// depending on which banks were passed. +/// +/// Note that this does not eagerly validate that the right accounts were passed. That +/// validation happens only when banks, perps etc are requested. pub fn new_fixed_order_account_retriever_with_optional_banks<'a, 'info>( ais: &'a [AccountInfo<'info>], account: &MangoAccountRef, @@ -180,9 +187,11 @@ impl FixedOrderAccountRetriever { Ok(market) } - fn oracle_price_perp(&self, account_index: usize, perp_market: &PerpMarket) -> Result { - let oracle = &self.ais[account_index]; - let oracle_acc_infos = OracleAccountInfos::from_reader(oracle); + fn oracle_price_perp( + &self, + oracle_acc_infos: &OracleAccountInfos, + perp_market: &PerpMarket, + ) -> Result { perp_market.oracle_price(&oracle_acc_infos, self.staleness_slot) } @@ -259,7 +268,9 @@ impl AccountRetriever for FixedOrderAccountRetriever { })?; let oracle_index = perp_index + self.n_perps; - let oracle_price = self.oracle_price_perp(oracle_index, perp_market).with_context(|| { + let oracle_acc_infos = + &self.create_oracle_infos(oracle_index, &perp_market.fallback_oracle); + let oracle_price = self.oracle_price_perp(oracle_acc_infos, perp_market).with_context(|| { format!( "getting oracle for perp market with health account index {} and perp market index {}, passed account {}", oracle_index, @@ -545,7 +556,17 @@ impl<'a, 'info> ScanningAccountRetriever<'a, 'info> { // The account was already loaded successfully during construction let perp_market = self.perp_markets[index].load_fully_unchecked::()?; let oracle_acc = &self.perp_oracles[index]; - let oracle_acc_infos = OracleAccountInfos::from_reader(oracle_acc); + + let fallback_opt = if &perp_market.fallback_oracle == &Pubkey::default() { + None + } else { + self.banks_and_oracles + .fallback_oracles + .iter() + .find(|ai| ai.key == &perp_market.fallback_oracle) + }; + let oracle_acc_infos = + OracleAccountInfos::from_reader_with_fallback(oracle_acc, fallback_opt); let price = perp_market.oracle_price(&oracle_acc_infos, self.banks_and_oracles.staleness_slot)?; Ok((perp_market, price)) diff --git a/programs/mango-v4/src/health/cache.rs b/programs/mango-v4/src/health/cache.rs index 6a105af36d..21f8cf2895 100644 --- a/programs/mango-v4/src/health/cache.rs +++ b/programs/mango-v4/src/health/cache.rs @@ -820,6 +820,12 @@ impl HealthCache { }) } + pub fn has_token_info(&self, token_index: TokenIndex) -> bool { + self.token_infos + .iter() + .any(|t| t.token_index == token_index) + } + pub fn perp_info(&self, perp_market_index: PerpMarketIndex) -> Result<&PerpInfo> { Ok(&self.perp_infos[self.perp_info_index(perp_market_index)?]) } @@ -1230,45 +1236,37 @@ pub fn new_health_cache( retriever: &impl AccountRetriever, now_ts: u64, ) -> Result { - new_health_cache_impl(account, retriever, now_ts, false, false) + new_health_cache_impl(account, retriever, now_ts, false) } /// Generate a special HealthCache for an account and its health accounts -/// where nonnegative token positions for bad oracles are skipped. +/// where nonnegative token positions for bad oracles are skipped as well as missing banks. /// /// This health cache must be used carefully, since it doesn't provide the actual /// account health, just a value that is guaranteed to be less than it. -pub fn new_health_cache_skipping_bad_oracles( - account: &MangoAccountRef, - retriever: &impl AccountRetriever, - now_ts: u64, -) -> Result { - new_health_cache_impl(account, retriever, now_ts, true, false) -} - pub fn new_health_cache_skipping_missing_banks_and_bad_oracles( account: &MangoAccountRef, retriever: &impl AccountRetriever, now_ts: u64, ) -> Result { - new_health_cache_impl(account, retriever, now_ts, true, true) + new_health_cache_impl(account, retriever, now_ts, true) } +// On `allow_skipping_banks`: +// If (a Bank is not provided or its oracle is stale or inconfident) and the health contribution would +// not be negative, skip it. This decreases health, but many operations are still allowed as long +// as the decreased amount stays positive. fn new_health_cache_impl( account: &MangoAccountRef, retriever: &impl AccountRetriever, now_ts: u64, - // If an oracle is stale or inconfident and the health contribution would - // not be negative, skip it. This decreases health, but maybe overall it's - // still positive? - skip_bad_oracles: bool, - skip_missing_banks: bool, + allow_skipping_banks: bool, ) -> Result { // token contribution from token accounts let mut token_infos = Vec::with_capacity(account.active_token_positions().count()); // As a CU optimization, don't call available_banks() unless necessary - let available_banks_opt = if skip_missing_banks { + let available_banks_opt = if allow_skipping_banks { Some(retriever.available_banks()?) } else { None @@ -1276,7 +1274,7 @@ fn new_health_cache_impl( for (i, position) in account.active_token_positions().enumerate() { // Allow skipping of missing banks only if the account has a nonnegative balance - if skip_missing_banks { + if allow_skipping_banks { let bank_is_available = available_banks_opt .as_ref() .unwrap() @@ -1296,7 +1294,7 @@ fn new_health_cache_impl( retriever.bank_and_oracle(&account.fixed.group, i, position.token_index); // Allow skipping of bad-oracle banks if the account has a nonnegative balance - if skip_bad_oracles + if allow_skipping_banks && bank_oracle_result.is_oracle_error() && position.indexed_position >= 0 { @@ -1345,7 +1343,7 @@ fn new_health_cache_impl( (Ok(base), Ok(quote)) => (base, quote), _ => { require_msg_typed!( - skip_bad_oracles || skip_missing_banks, + allow_skipping_banks, MangoError::InvalidBank, "serum market {} misses health accounts for bank {} or {}", serum_account.market_index, @@ -1382,7 +1380,7 @@ fn new_health_cache_impl( )?; // Ensure the settle token is available in the health cache - if skip_bad_oracles || skip_missing_banks { + if allow_skipping_banks { find_token_info_index(&token_infos, perp_market.settle_token_index)?; } diff --git a/programs/mango-v4/src/instructions/perp_create_market.rs b/programs/mango-v4/src/instructions/perp_create_market.rs index 4415688dc5..7cf4450351 100644 --- a/programs/mango-v4/src/instructions/perp_create_market.rs +++ b/programs/mango-v4/src/instructions/perp_create_market.rs @@ -58,6 +58,7 @@ pub fn perp_create_market( event_queue: ctx.accounts.event_queue.key(), oracle: ctx.accounts.oracle.key(), oracle_config: oracle_config.to_oracle_config(), + fallback_oracle: Pubkey::default(), stable_price_model: StablePriceModel::default(), quote_lot_size, base_lot_size, @@ -95,7 +96,7 @@ pub fn perp_create_market( fees_withdrawn: 0, platform_liquidation_fee: I80F48::from_num(platform_liquidation_fee), accrued_liquidation_fees: I80F48::ZERO, - reserved: [0; 1848], + reserved: [0; 1816], }; let oracle_ref = &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?; diff --git a/programs/mango-v4/src/instructions/perp_edit_market.rs b/programs/mango-v4/src/instructions/perp_edit_market.rs index 821f7ab530..5d2378e554 100644 --- a/programs/mango-v4/src/instructions/perp_edit_market.rs +++ b/programs/mango-v4/src/instructions/perp_edit_market.rs @@ -40,6 +40,7 @@ pub fn perp_edit_market( name_opt: Option, force_close_opt: Option, platform_liquidation_fee_opt: Option, + set_fallback_oracle: bool, ) -> Result<()> { let group = ctx.accounts.group.load()?; @@ -63,6 +64,18 @@ pub fn perp_edit_market( perp_market.oracle = oracle; require_group_admin = true; } + if set_fallback_oracle { + msg!( + "Fallback oracle old {:?}, new {:?}", + perp_market.fallback_oracle, + ctx.accounts.fallback_oracle.key() + ); + check_is_valid_fallback_oracle(&AccountInfoRef::borrow( + ctx.accounts.fallback_oracle.as_ref(), + )?)?; + perp_market.fallback_oracle = ctx.accounts.fallback_oracle.key(); + require_group_admin = true; + } if reset_stable_price { msg!("Stable price reset"); require_keys_eq!(perp_market.oracle, ctx.accounts.oracle.key()); diff --git a/programs/mango-v4/src/instructions/perp_force_close_position.rs b/programs/mango-v4/src/instructions/perp_force_close_position.rs index 494136dc0b..67a80060a5 100644 --- a/programs/mango-v4/src/instructions/perp_force_close_position.rs +++ b/programs/mango-v4/src/instructions/perp_force_close_position.rs @@ -35,8 +35,18 @@ pub fn perp_force_close_position(ctx: Context) -> Result .max(0); let now_slot = Clock::get()?.slot; let oracle_ref = &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?; - let oracle_price = - perp_market.oracle_price(&OracleAccountInfos::from_reader(oracle_ref), Some(now_slot))?; + let fallback_opt = if perp_market.fallback_oracle != Pubkey::default() { + ctx.remaining_accounts + .iter() + .find(|a| a.key == &perp_market.fallback_oracle) + .map(|k| AccountInfoRef::borrow(k).unwrap()) + } else { + None + }; + let oracle_price = perp_market.oracle_price( + &OracleAccountInfos::from_reader_with_fallback(oracle_ref, fallback_opt.as_ref()), + Some(now_slot), + )?; let quote_transfer = I80F48::from(base_transfer * perp_market.base_lot_size) * oracle_price; account_a_perp_position.record_trade(&mut perp_market, -base_transfer, quote_transfer); diff --git a/programs/mango-v4/src/instructions/perp_liq_base_or_positive_pnl.rs b/programs/mango-v4/src/instructions/perp_liq_base_or_positive_pnl.rs index 54d1916561..941a32aa4c 100644 --- a/programs/mango-v4/src/instructions/perp_liq_base_or_positive_pnl.rs +++ b/programs/mango-v4/src/instructions/perp_liq_base_or_positive_pnl.rs @@ -70,8 +70,16 @@ pub fn perp_liq_base_or_positive_pnl( // Get oracle price for market. Price is validated inside let oracle_ref = &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?; + let fallback_opt = if perp_market.fallback_oracle != Pubkey::default() { + ctx.remaining_accounts + .iter() + .find(|a| a.key == &perp_market.fallback_oracle) + .map(|k| AccountInfoRef::borrow(k).unwrap()) + } else { + None + }; let oracle_price = perp_market.oracle_price( - &OracleAccountInfos::from_reader(oracle_ref), + &OracleAccountInfos::from_reader_with_fallback(oracle_ref, fallback_opt.as_ref()), None, // checked in health )?; diff --git a/programs/mango-v4/src/instructions/perp_liq_negative_pnl_or_bankruptcy.rs b/programs/mango-v4/src/instructions/perp_liq_negative_pnl_or_bankruptcy.rs index 49b8416f9c..140260d6a3 100644 --- a/programs/mango-v4/src/instructions/perp_liq_negative_pnl_or_bankruptcy.rs +++ b/programs/mango-v4/src/instructions/perp_liq_negative_pnl_or_bankruptcy.rs @@ -34,23 +34,55 @@ pub fn perp_liq_negative_pnl_or_bankruptcy( perp_market_index = perp_market.perp_market_index; settle_token_index = perp_market.settle_token_index; let oracle_ref = &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?; - perp_oracle_price = perp_market - .oracle_price(&OracleAccountInfos::from_reader(oracle_ref), Some(now_slot))?; + let fallback_opt = if perp_market.fallback_oracle != Pubkey::default() { + ctx.remaining_accounts + .iter() + .find(|a| a.key == &perp_market.fallback_oracle) + .map(|k| AccountInfoRef::borrow(k).unwrap()) + } else { + None + }; + perp_oracle_price = perp_market.oracle_price( + &OracleAccountInfos::from_reader_with_fallback(oracle_ref, fallback_opt.as_ref()), + Some(now_slot), + )?; let settle_bank = ctx.accounts.settle_bank.load()?; let settle_oracle_ref = &AccountInfoRef::borrow(ctx.accounts.settle_oracle.as_ref())?; + let settle_fallback_opt = if settle_bank.fallback_oracle != Pubkey::default() { + ctx.remaining_accounts + .iter() + .find(|a| a.key == &settle_bank.fallback_oracle) + .map(|k| AccountInfoRef::borrow(k).unwrap()) + } else { + None + }; settle_token_oracle_price = settle_bank.oracle_price( - &OracleAccountInfos::from_reader(settle_oracle_ref), + &OracleAccountInfos::from_reader_with_fallback( + settle_oracle_ref, + settle_fallback_opt.as_ref(), + ), Some(now_slot), )?; drop(settle_bank); // could be the same as insurance_bank let insurance_bank = ctx.accounts.insurance_bank.load()?; let insurance_oracle_ref = &AccountInfoRef::borrow(ctx.accounts.insurance_oracle.as_ref())?; + let insurance_fallback_opt = if insurance_bank.fallback_oracle != Pubkey::default() { + ctx.remaining_accounts + .iter() + .find(|a| a.key == &insurance_bank.fallback_oracle) + .map(|k| AccountInfoRef::borrow(k).unwrap()) + } else { + None + }; // We're not getting the insurance token price from the HealthCache because // the liqee isn't guaranteed to have an insurance fund token position. insurance_token_oracle_price = insurance_bank.oracle_price( - &OracleAccountInfos::from_reader(insurance_oracle_ref), + &OracleAccountInfos::from_reader_with_fallback( + insurance_oracle_ref, + insurance_fallback_opt.as_ref(), + ), Some(now_slot), )?; } diff --git a/programs/mango-v4/src/instructions/perp_place_order.rs b/programs/mango-v4/src/instructions/perp_place_order.rs index 36e3b95797..2cddf102ec 100644 --- a/programs/mango-v4/src/instructions/perp_place_order.rs +++ b/programs/mango-v4/src/instructions/perp_place_order.rs @@ -31,9 +31,18 @@ pub fn perp_place_order( asks: ctx.accounts.asks.load_mut()?, }; + let fallback_opt = if perp_market.fallback_oracle != Pubkey::default() { + ctx.remaining_accounts + .iter() + .find(|a| a.key == &perp_market.fallback_oracle) + .map(|k| AccountInfoRef::borrow(k).unwrap()) + } else { + None + }; + let oracle_ref = &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?; let oracle_state = perp_market.oracle_state( - &OracleAccountInfos::from_reader(oracle_ref), + &OracleAccountInfos::from_reader_with_fallback(oracle_ref, fallback_opt.as_ref()), None, // staleness checked in health )?; oracle_price = oracle_state.price; diff --git a/programs/mango-v4/src/instructions/token_withdraw.rs b/programs/mango-v4/src/instructions/token_withdraw.rs index 7f32940f50..28d85dc375 100644 --- a/programs/mango-v4/src/instructions/token_withdraw.rs +++ b/programs/mango-v4/src/instructions/token_withdraw.rs @@ -13,8 +13,6 @@ use crate::logs::{ emit_stack, LoanOriginationFeeInstruction, TokenBalanceLog, WithdrawLoanLog, WithdrawLog, }; -const DELEGATE_WITHDRAW_MAX: i64 = 100_000; // $0.1 - pub fn token_withdraw(ctx: Context, amount: u64, allow_borrow: bool) -> Result<()> { require_msg!(amount > 0, "withdraw amount must be positive"); @@ -143,20 +141,13 @@ pub fn token_withdraw(ctx: Context, amount: u64, allow_borrow: bo !withdraw_result.position_is_active, MangoError::DelegateWithdrawMustClosePosition ); - - // Delegates can't withdraw too much - require_gte!( - DELEGATE_WITHDRAW_MAX, - amount_usd, - MangoError::DelegateWithdrawSmall - ); } // // Health check // if let Some((mut health_cache, pre_init_health_lower_bound)) = pre_health_opt { - if health_cache.token_info_index(token_index).is_ok() { + if health_cache.has_token_info(token_index) { // This is the normal case: the health cache knows about the token, we can // compute the health for the new state by adjusting its balance health_cache.adjust_token_balance(&bank, native_position_after - native_position)?; diff --git a/programs/mango-v4/src/lib.rs b/programs/mango-v4/src/lib.rs index 533434b148..4608cecc80 100644 --- a/programs/mango-v4/src/lib.rs +++ b/programs/mango-v4/src/lib.rs @@ -946,6 +946,7 @@ pub mod mango_v4 { name_opt: Option, force_close_opt: Option, platform_liquidation_fee_opt: Option, + set_fallback_oracle: bool, ) -> Result<()> { #[cfg(feature = "enable-gpl")] instructions::perp_edit_market( @@ -981,6 +982,7 @@ pub mod mango_v4 { name_opt, force_close_opt, platform_liquidation_fee_opt, + set_fallback_oracle, )?; Ok(()) } diff --git a/programs/mango-v4/src/state/oracle.rs b/programs/mango-v4/src/state/oracle.rs index 64ce6327fe..aa08c1590b 100644 --- a/programs/mango-v4/src/state/oracle.rs +++ b/programs/mango-v4/src/state/oracle.rs @@ -295,6 +295,15 @@ impl<'a, T: KeyedAccountReader> OracleAccountInfos<'a, T> { sol_opt: None, } } + + pub fn from_reader_with_fallback(acc_reader: &'a T, fallback_opt: Option<&'a T>) -> Self { + OracleAccountInfos { + oracle: acc_reader, + fallback_opt, + usdc_opt: None, + sol_opt: None, + } + } } /// Returns the price of one native base token, in native quote tokens diff --git a/programs/mango-v4/src/state/perp_market.rs b/programs/mango-v4/src/state/perp_market.rs index 2b1c795a3d..0cfd6a6c1a 100644 --- a/programs/mango-v4/src/state/perp_market.rs +++ b/programs/mango-v4/src/state/perp_market.rs @@ -8,7 +8,7 @@ use oracle::oracle_log_context; use static_assertions::const_assert_eq; use crate::accounts_zerocopy::KeyedAccountReader; -use crate::error::{Contextable, MangoError}; +use crate::error::{Contextable, IsAnchorErrorWithCode, MangoError}; use crate::logs::{emit_stack, PerpUpdateFundingLogV2}; use crate::state::orderbook::Side; use crate::state::{oracle, TokenIndex}; @@ -198,8 +198,12 @@ pub struct PerpMarket { /// liquidation fees that happened. So never decreases (different to fees_accrued). pub accrued_liquidation_fees: I80F48, + /// Oracle that may be used if the main oracle is stale or not confident enough. + /// If this is Pubkey::default(), no fallback is available. + pub fallback_oracle: Pubkey, + #[derivative(Debug = "ignore")] - pub reserved: [u8; 1848], + pub reserved: [u8; 1816], } const_assert_eq!( @@ -237,7 +241,8 @@ const_assert_eq!( + 3 * 16 + 8 + 2 * 16 - + 1848 + + 32 + + 1816 ); const_assert_eq!(size_of::(), 2808); const_assert_eq!(size_of::() % 8, 0); @@ -288,13 +293,45 @@ impl PerpMarket { staleness_slot: Option, ) -> Result { require_keys_eq!(self.oracle, *oracle_acc_infos.oracle.key()); - let state = oracle::oracle_state_unchecked(oracle_acc_infos, self.base_decimals)?; - state - .check_confidence_and_maybe_staleness(&self.oracle_config, staleness_slot) - .with_context(|| { - oracle_log_context(self.name(), &state, &self.oracle_config, staleness_slot) + let primary_state = oracle::oracle_state_unchecked(oracle_acc_infos, self.base_decimals)?; + let primary_ok = + primary_state.check_confidence_and_maybe_staleness(&self.oracle_config, staleness_slot); + if primary_ok.is_oracle_error() && oracle_acc_infos.fallback_opt.is_some() { + let fallback_oracle_acc = oracle_acc_infos.fallback_opt.unwrap(); + require_keys_eq!(self.fallback_oracle, *fallback_oracle_acc.key()); + let fallback_state = + oracle::fallback_oracle_state_unchecked(&oracle_acc_infos, self.base_decimals)?; + let fallback_ok = fallback_state + .check_confidence_and_maybe_staleness(&self.oracle_config, staleness_slot); + fallback_ok.with_context(|| { + format!( + "{} {}", + oracle_log_context( + self.name(), + &primary_state, + &self.oracle_config, + staleness_slot + ), + oracle_log_context( + self.name(), + &fallback_state, + &self.oracle_config, + staleness_slot + ) + ) })?; - Ok(state) + Ok(fallback_state) + } else { + primary_ok.with_context(|| { + oracle_log_context( + self.name(), + &primary_state, + &self.oracle_config, + staleness_slot, + ) + })?; + Ok(primary_state) + } } pub fn stable_price(&self) -> I80F48 { @@ -500,6 +537,7 @@ impl PerpMarket { max_staleness_slots: -1, reserved: [0; 72], }, + fallback_oracle: Pubkey::default(), stable_price_model: StablePriceModel::default(), quote_lot_size: 1, base_lot_size: 1, @@ -537,7 +575,7 @@ impl PerpMarket { fees_withdrawn: 0, platform_liquidation_fee: I80F48::ZERO, accrued_liquidation_fees: I80F48::ZERO, - reserved: [0; 1848], + reserved: [0; 1816], } } } diff --git a/programs/mango-v4/tests/cases/test_health_check.rs b/programs/mango-v4/tests/cases/test_health_check.rs index b4624218b6..bb0c792a3a 100644 --- a/programs/mango-v4/tests/cases/test_health_check.rs +++ b/programs/mango-v4/tests/cases/test_health_check.rs @@ -7,8 +7,6 @@ use mango_v4::accounts_ix::{HealthCheck, HealthCheckKind}; use mango_v4::error::MangoError; use solana_sdk::transport::TransportError; -// TODO FAS - #[tokio::test] async fn test_health_check() -> Result<(), TransportError> { let context = TestContext::new().await; diff --git a/programs/mango-v4/tests/cases/test_liq_perps_bankruptcy.rs b/programs/mango-v4/tests/cases/test_liq_perps_bankruptcy.rs index d551cb4eda..623450e94a 100644 --- a/programs/mango-v4/tests/cases/test_liq_perps_bankruptcy.rs +++ b/programs/mango-v4/tests/cases/test_liq_perps_bankruptcy.rs @@ -1,4 +1,5 @@ use super::*; +use anchor_lang::prelude::AccountMeta; #[tokio::test] async fn test_liq_perps_bankruptcy() -> Result<(), TransportError> { @@ -450,3 +451,422 @@ async fn test_liq_perps_bankruptcy() -> Result<(), TransportError> { Ok(()) } + +#[tokio::test] +/// Copy of the above test with an added fallback oracle + staleness instructions +async fn test_liq_perps_bankruptcy_stale_oracle() -> Result<(), TransportError> { + let mut test_builder = TestContextBuilder::new(); + test_builder.test().set_compute_max_units(200_000); // PerpLiqNegativePnlOrBankruptcy takes a lot of CU + let context = test_builder.start_default().await; + let solana = &context.solana.clone(); + + let admin = TestKeypair::new(); + let owner = context.users[0].key; + let payer = context.users[1].key; + let mints = &context.mints[0..3]; + let payer_mint_accounts = &context.users[1].token_accounts[0..3]; + + // + // SETUP: Create a group and an account to fill the vaults + // + + let GroupWithTokens { + group, + tokens, + insurance_vault, + .. + } = GroupWithTokensConfig { + admin, + payer, + mints: mints.to_vec(), + zero_token_is_quote: true, + ..GroupWithTokensConfig::default() + } + .create(solana) + .await; + + send_tx( + solana, + TokenEditWeights { + group, + admin, + mint: mints[2].pubkey, + maint_liab_weight: 1.0, + maint_asset_weight: 1.0, + init_liab_weight: 1.0, + init_asset_weight: 1.0, + }, + ) + .await + .unwrap(); + + let fund_insurance = |amount: u64| async move { + let mut tx = ClientTransaction::new(solana); + tx.add_instruction_direct( + spl_token::instruction::transfer( + &spl_token::ID, + &payer_mint_accounts[0], + &insurance_vault, + &payer.pubkey(), + &[&payer.pubkey()], + amount, + ) + .unwrap(), + ); + tx.add_signer(payer); + tx.send().await.unwrap(); + }; + + let quote_token = &tokens[0]; // USDC, 1/1 weights, price 1, never changed + let base_token = &tokens[1]; // used for perp market + let collateral_token = &tokens[2]; // used for adjusting account health + + // all perp markets used here default to price = 1.0, base_lot_size = 100 + let price_lots = 100; + + let context_ref = &context; + let mut perp_market_index: PerpMarketIndex = 0; + let setup_perp_inner = |perp_market_index: PerpMarketIndex, + health: i64, + pnl: i64, + settle_limit: i64| async move { + // price used later to produce negative pnl with a short: + // doubling the price leads to -100 pnl + let adj_price = 1.0 + pnl as f64 / -100.0; + let adj_price_lots = (price_lots as f64 * adj_price) as i64; + + let fresh_liqor = create_funded_account( + &solana, + group, + owner, + 200 + perp_market_index as u32, + &context_ref.users[1], + mints, + 10000, + 0, + ) + .await; + + let mango_v4::accounts::PerpCreateMarket { perp_market, .. } = send_tx( + solana, + PerpCreateMarketInstruction { + group, + admin, + payer, + perp_market_index, + quote_lot_size: 1, + base_lot_size: 100, + maint_base_asset_weight: 0.8, + init_base_asset_weight: 0.6, + maint_base_liab_weight: 1.2, + init_base_liab_weight: 1.4, + base_liquidation_fee: 0.05, + maker_fee: 0.0, + taker_fee: 0.0, + group_insurance_fund: true, + // adjust this factur such that we get the desired settle limit in the end + settle_pnl_limit_factor: (settle_limit as f32 - 0.1).max(0.0) + / (1.0 * 100.0 * adj_price) as f32, + settle_pnl_limit_window_size_ts: 24 * 60 * 60, + ..PerpCreateMarketInstruction::with_new_book_and_queue(&solana, base_token).await + }, + ) + .await + .unwrap(); + set_perp_stub_oracle_price(solana, group, perp_market, &base_token, admin, 1.0).await; + set_bank_stub_oracle_price(solana, group, &collateral_token, admin, 1.0).await; + + // + // SETUP: accounts + // + let deposit_amount = 1000; + let helper_account = create_funded_account( + &solana, + group, + owner, + perp_market_index as u32 * 2, + &context_ref.users[1], + &mints[2..3], + deposit_amount, + 0, + ) + .await; + let account = create_funded_account( + &solana, + group, + owner, + perp_market_index as u32 * 2 + 1, + &context_ref.users[1], + &mints[2..3], + deposit_amount, + 0, + ) + .await; + + // + // SETUP: Trade perps between accounts twice to generate pnl, settle_limit + // + let mut tx = ClientTransaction::new(solana); + tx.add_instruction(PerpPlaceOrderInstruction { + account: helper_account, + perp_market, + owner, + side: Side::Bid, + price_lots, + max_base_lots: 1, + ..PerpPlaceOrderInstruction::default() + }) + .await; + tx.add_instruction(PerpPlaceOrderInstruction { + account: account, + perp_market, + owner, + side: Side::Ask, + price_lots, + max_base_lots: 1, + ..PerpPlaceOrderInstruction::default() + }) + .await; + tx.add_instruction(PerpConsumeEventsInstruction { + perp_market, + mango_accounts: vec![account, helper_account], + }) + .await; + tx.send().await.unwrap(); + + set_perp_stub_oracle_price(solana, group, perp_market, &base_token, admin, adj_price).await; + let mut tx = ClientTransaction::new(solana); + tx.add_instruction(PerpPlaceOrderInstruction { + account: helper_account, + perp_market, + owner, + side: Side::Ask, + price_lots: adj_price_lots, + max_base_lots: 1, + ..PerpPlaceOrderInstruction::default() + }) + .await; + tx.add_instruction(PerpPlaceOrderInstruction { + account: account, + perp_market, + owner, + side: Side::Bid, + price_lots: adj_price_lots, + max_base_lots: 1, + ..PerpPlaceOrderInstruction::default() + }) + .await; + tx.add_instruction(PerpConsumeEventsInstruction { + perp_market, + mango_accounts: vec![account, helper_account], + }) + .await; + tx.send().await.unwrap(); + + set_perp_stub_oracle_price(solana, group, perp_market, &base_token, admin, 1.0).await; + + // Adjust target health: + // full health = 1000 * collat price * 1.0 + pnl + set_bank_stub_oracle_price( + solana, + group, + &collateral_token, + admin, + (health - pnl) as f64 / 1000.0, + ) + .await; + + // Verify we got it right + let account_data = solana.get_account::(account).await; + assert_eq!(account_data.perps[0].quote_position_native(), pnl); + assert_eq!( + account_data.perps[0].recurring_settle_pnl_allowance, + settle_limit + ); + assert_eq!( + account_init_health(solana, account).await.round(), + health as f64 + ); + + (perp_market, account, fresh_liqor) + }; + let mut setup_perp = |health: i64, pnl: i64, settle_limit: i64| { + let out = setup_perp_inner(perp_market_index, health, pnl, settle_limit); + perp_market_index += 1; + out + }; + + let limit_prec = |f: f64| (f * 1000.0).round() / 1000.0; + + let liq_event_amounts = || { + let settlement = solana + .program_log_events::() + .pop() + .map(|v| limit_prec(I80F48::from_bits(v.settlement).to_num::())) + .unwrap_or(0.0); + let (insur, loss) = solana + .program_log_events::() + .pop() + .map(|v| { + ( + I80F48::from_bits(v.insurance_transfer).to_num::(), + limit_prec(I80F48::from_bits(v.socialized_loss).to_num::()), + ) + }) + .unwrap_or((0, 0.0)); + (settlement, insur, loss) + }; + + let liqor_info = |perp_market: Pubkey, liqor: Pubkey| async move { + let perp_market = solana.get_account::(perp_market).await; + let liqor_data = solana.get_account::(liqor).await; + let liqor_perp = liqor_data + .perps + .iter() + .find(|p| p.market_index == perp_market.perp_market_index) + .unwrap() + .clone(); + (liqor_data, liqor_perp) + }; + + { + let (perp_market, account, liqor) = setup_perp(-28, -50, 10).await; + let liqor_quote_before = account_position(solana, liqor, quote_token.bank).await; + + // + // SETUP: Fallback oracle + // + let fallback_oracle_kp = TestKeypair::new(); + let fallback_oracle = fallback_oracle_kp.pubkey(); + send_tx( + solana, + StubOracleCreate { + oracle: fallback_oracle_kp, + group, + mint: base_token.mint.pubkey, + admin, + payer, + }, + ) + .await + .unwrap(); + + send_tx( + solana, + PerpAddFallbackOracle { + group, + admin, + perp_market, + fallback_oracle, + }, + ) + .await + .unwrap(); + + send_tx( + solana, + TokenEdit { + group, + admin, + mint: base_token.mint.pubkey, + fallback_oracle, + options: mango_v4::instruction::TokenEdit { + set_fallback_oracle: true, + ..token_edit_instruction_default() + }, + }, + ) + .await + .unwrap(); + + // + // SETUP: Change the oracle to be invalid + // + send_tx( + solana, + StubOracleSetTestInstruction { + oracle: base_token.oracle, + group, + mint: base_token.mint.pubkey, + admin, + price: 1.0, + last_update_slot: 0, + deviation: 100.0, + }, + ) + .await + .unwrap(); + + // + // SETUP: Ensure fallback oracle matches default + // + send_tx( + solana, + StubOracleSetTestInstruction { + oracle: fallback_oracle, + group, + mint: base_token.mint.pubkey, + admin, + price: 1.0, + last_update_slot: 0, + deviation: 0.0, + }, + ) + .await + .unwrap(); + + assert!(send_tx( + solana, + PerpLiqNegativePnlOrBankruptcyInstruction { + liqor, + liqor_owner: owner, + liqee: account, + perp_market, + max_liab_transfer: 1, + }, + ) + .await + .is_err()); + + // + // TEST: Liq with fallback succeeds + // + let fallback_oracle_meta = AccountMeta { + pubkey: fallback_oracle, + is_writable: false, + is_signer: false, + }; + assert!(send_tx_with_extra_accounts( + solana, + PerpLiqNegativePnlOrBankruptcyInstruction { + liqor, + liqor_owner: owner, + liqee: account, + perp_market, + max_liab_transfer: 1, + }, + vec![fallback_oracle_meta], + ) + .await + .unwrap() + .result + .is_ok()); + assert_eq!(liq_event_amounts(), (1.0, 0, 0.0)); + + assert_eq!( + account_position(solana, account, quote_token.bank).await, + -1 + ); + assert_eq!( + account_position(solana, liqor, quote_token.bank).await, + liqor_quote_before + 1 + ); + let acc_data = solana.get_account::(account).await; + assert_eq!(acc_data.perps[0].quote_position_native(), -49); + assert_eq!(acc_data.being_liquidated, 1); + let (_liqor_data, liqor_perp) = liqor_info(perp_market, liqor).await; + assert_eq!(liqor_perp.quote_position_native(), -1); + } + + Ok(()) +} diff --git a/programs/mango-v4/tests/cases/test_liq_perps_force_cancel.rs b/programs/mango-v4/tests/cases/test_liq_perps_force_cancel.rs index f30fd2b55e..6a6bb43d3e 100644 --- a/programs/mango-v4/tests/cases/test_liq_perps_force_cancel.rs +++ b/programs/mango-v4/tests/cases/test_liq_perps_force_cancel.rs @@ -1,4 +1,5 @@ use super::*; +use anchor_lang::prelude::AccountMeta; #[tokio::test] async fn test_liq_perps_force_cancel() -> Result<(), TransportError> { @@ -162,3 +163,266 @@ async fn test_liq_perps_force_cancel() -> Result<(), TransportError> { Ok(()) } + +#[tokio::test] +async fn test_liq_perps_force_cancel_stale_oracle() -> Result<(), TransportError> { + let mut test_builder = TestContextBuilder::new(); + test_builder.test().set_compute_max_units(150_000); // bad oracles log a lot + let context = test_builder.start_default().await; + let solana = &context.solana.clone(); + + let admin = TestKeypair::new(); + let owner = context.users[0].key; + let payer = context.users[1].key; + let mints = &context.mints[0..2]; + let payer_mint_accounts = &context.users[1].token_accounts[0..2]; + + // + // SETUP: Create a group and an account to fill the vaults + // + + let GroupWithTokens { group, tokens, .. } = GroupWithTokensConfig { + admin, + payer, + mints: mints.to_vec(), + ..GroupWithTokensConfig::default() + } + .create(solana) + .await; + //let quote_token = &tokens[0]; + let base_token = &tokens[1]; + + // deposit some funds, to the vaults aren't empty + create_funded_account(&solana, group, owner, 0, &context.users[1], mints, 10000, 0).await; + + // + // TEST: Create a perp market + // + let mango_v4::accounts::PerpCreateMarket { perp_market, .. } = send_tx( + solana, + PerpCreateMarketInstruction { + group, + admin, + payer, + perp_market_index: 0, + quote_lot_size: 10, + base_lot_size: 100, + maint_base_asset_weight: 0.8, + init_base_asset_weight: 0.6, + maint_base_liab_weight: 1.2, + init_base_liab_weight: 1.4, + base_liquidation_fee: 0.05, + maker_fee: 0.0, + taker_fee: 0.0, + settle_pnl_limit_factor: 0.2, + settle_pnl_limit_window_size_ts: 24 * 60 * 60, + ..PerpCreateMarketInstruction::with_new_book_and_queue(&solana, base_token).await + }, + ) + .await + .unwrap(); + + let price_lots = { + let perp_market = solana.get_account::(perp_market).await; + perp_market.native_price_to_lot(I80F48::ONE) + }; + + // + // SETUP: Make an account and deposit some quote and base + // + let deposit_amount = 1000; + let account = create_funded_account( + &solana, + group, + owner, + 1, + &context.users[1], + &mints[0..1], + deposit_amount, + 0, + ) + .await; + + send_tx( + solana, + TokenDepositInstruction { + amount: 1, + reduce_only: false, + account, + owner, + token_account: payer_mint_accounts[1], + token_authority: payer, + bank_index: 0, + }, + ) + .await + .unwrap(); + + // + // SETUP: Fallback oracle + // + let fallback_oracle_kp = TestKeypair::new(); + let fallback_oracle = fallback_oracle_kp.pubkey(); + send_tx( + solana, + StubOracleCreate { + oracle: fallback_oracle_kp, + group, + mint: base_token.mint.pubkey, + admin, + payer, + }, + ) + .await + .unwrap(); + + send_tx( + solana, + PerpAddFallbackOracle { + group, + admin, + perp_market, + fallback_oracle, + }, + ) + .await + .unwrap(); + + send_tx( + solana, + TokenEdit { + group, + admin, + mint: base_token.mint.pubkey, + fallback_oracle, + options: mango_v4::instruction::TokenEdit { + set_fallback_oracle: true, + ..token_edit_instruction_default() + }, + }, + ) + .await + .unwrap(); + + // + // SETUP: Place a perp order + // + send_tx( + solana, + PerpPlaceOrderInstruction { + account, + perp_market, + owner, + side: Side::Ask, + price_lots, + // health was 1000 * 0.6 = 600; this order is -14*100*(1.4-1) = -560 + max_base_lots: 14, + ..PerpPlaceOrderInstruction::default() + }, + ) + .await + .unwrap(); + + // + // SETUP: Change the oracle to make health go negative, and invalid + // + send_tx( + solana, + StubOracleSetTestInstruction { + oracle: base_token.oracle, + group, + mint: base_token.mint.pubkey, + admin, + price: 10.0, + last_update_slot: 0, + deviation: 100.0, + }, + ) + .await + .unwrap(); + + // + // TEST: force cancel orders fails due to stale oracle + // + assert!(send_tx( + solana, + PerpLiqForceCancelOrdersInstruction { + account, + perp_market, + }, + ) + .await + .is_err()); + + // + // SETUP: Ensure fallback oracle matches default + // + send_tx( + solana, + StubOracleSetTestInstruction { + oracle: fallback_oracle, + group, + mint: base_token.mint.pubkey, + admin, + price: 10.0, + last_update_slot: 0, + deviation: 0.0, + }, + ) + .await + .unwrap(); + + // + // TEST: force cancel orders with fallback succeeds + // + let fallback_oracle_meta = AccountMeta { + pubkey: fallback_oracle, + is_writable: false, + is_signer: false, + }; + send_tx_with_extra_accounts( + solana, + PerpLiqForceCancelOrdersInstruction { + account, + perp_market, + }, + vec![fallback_oracle_meta.clone()], + ) + .await + .unwrap(); + + // Withdraw also fails due to stale oracle + assert!(send_tx( + solana, + TokenWithdrawInstruction { + amount: 1, + allow_borrow: false, + account, + owner, + token_account: payer_mint_accounts[1], + bank_index: 0, + }, + ) + .await + .is_err()); + + // can withdraw with fallback + assert!(send_tx_with_extra_accounts( + solana, + TokenWithdrawInstruction { + amount: 1, + allow_borrow: false, + account, + owner, + token_account: payer_mint_accounts[1], + bank_index: 0, + }, + vec![fallback_oracle_meta.clone()], + ) + .await + .unwrap() + .result + .is_ok()); + + Ok(()) +} diff --git a/programs/mango-v4/tests/cases/test_liq_perps_positive_pnl.rs b/programs/mango-v4/tests/cases/test_liq_perps_positive_pnl.rs index cf734f1f24..7fa82ebf03 100644 --- a/programs/mango-v4/tests/cases/test_liq_perps_positive_pnl.rs +++ b/programs/mango-v4/tests/cases/test_liq_perps_positive_pnl.rs @@ -1,4 +1,5 @@ use super::*; +use anchor_lang::prelude::AccountMeta; #[tokio::test] async fn test_liq_perps_positive_pnl() -> Result<(), TransportError> { @@ -409,3 +410,368 @@ async fn test_liq_perps_positive_pnl() -> Result<(), TransportError> { Ok(()) } + +#[tokio::test] +async fn test_liq_perps_positive_pnl_stale_oracle() -> Result<(), TransportError> { + let mut test_builder = TestContextBuilder::new(); + test_builder.test().set_compute_max_units(170_000); // PerpLiqBaseOrPositivePnlInstruction takes a lot of CU + let context = test_builder.start_default().await; + let solana = &context.solana.clone(); + + let admin = TestKeypair::new(); + let owner = context.users[0].key; + let payer = context.users[1].key; + let mints = &context.mints[0..4]; + let payer_mint_accounts = &context.users[1].token_accounts[0..4]; + + // + // SETUP: Create a group and an account to fill the vaults + // + + let GroupWithTokens { + group, + tokens, + insurance_vault, + .. + } = GroupWithTokensConfig { + admin, + payer, + mints: mints.to_vec(), + zero_token_is_quote: true, + ..GroupWithTokensConfig::default() + } + .create(solana) + .await; + + // fund the insurance vault + let insurance_vault_funding = 100; + { + let mut tx = ClientTransaction::new(solana); + tx.add_instruction_direct( + spl_token::instruction::transfer( + &spl_token::ID, + &payer_mint_accounts[0], + &insurance_vault, + &payer.pubkey(), + &[&payer.pubkey()], + insurance_vault_funding, + ) + .unwrap(), + ); + tx.add_signer(payer); + tx.send().await.unwrap(); + } + + let _quote_token = &tokens[0]; + let base_token = &tokens[1]; + let borrow_token = &tokens[2]; + let settle_token = &tokens[3]; + + // deposit some funds, to the vaults aren't empty + let liqor = create_funded_account( + &solana, + group, + owner, + 250, + &context.users[1], + mints, + 10000, + 0, + ) + .await; + + // + // SETUP: Create a perp market + // + let mango_v4::accounts::PerpCreateMarket { perp_market, .. } = send_tx( + solana, + PerpCreateMarketInstruction { + group, + admin, + payer, + perp_market_index: 0, + settle_token_index: 3, + quote_lot_size: 10, + base_lot_size: 100, + maint_base_asset_weight: 0.8, + init_base_asset_weight: 0.5, + maint_base_liab_weight: 1.2, + init_base_liab_weight: 1.5, + maint_overall_asset_weight: 0.0, + init_overall_asset_weight: 0.0, + base_liquidation_fee: 0.05, + positive_pnl_liquidation_fee: 0.05, + maker_fee: 0.0, + taker_fee: 0.0, + group_insurance_fund: true, + settle_pnl_limit_factor: 0.2, + settle_pnl_limit_window_size_ts: 24 * 60 * 60, + ..PerpCreateMarketInstruction::with_new_book_and_queue(&solana, base_token).await + }, + ) + .await + .unwrap(); + + set_perp_stub_oracle_price(solana, group, perp_market, &base_token, admin, 10.0).await; + let price_lots = { + let perp_market = solana.get_account::(perp_market).await; + perp_market.native_price_to_lot(I80F48::from(10)) + }; + + // + // SETUP: Make an two accounts and deposit some quote and base + // + let context_ref = &context; + let make_account = |idx: u32| async move { + let deposit_amount = 10000; + let account = create_funded_account( + &solana, + group, + owner, + idx, + &context_ref.users[1], + &mints[0..1], + deposit_amount, + 0, + ) + .await; + + account + }; + let account_0 = make_account(0).await; + let account_1 = make_account(1).await; + + // + // SETUP: Borrow some spot on account_0, so we can later make it liquidatable that way + // (actually borrowing 1000.5 due to loan origination!) + // + send_tx( + solana, + TokenWithdrawInstruction { + amount: 1000, + allow_borrow: true, + account: account_0, + owner, + token_account: payer_mint_accounts[2], + bank_index: 0, + }, + ) + .await + .unwrap(); + + // + // SETUP: Trade perps between accounts + // + send_tx( + solana, + PerpPlaceOrderInstruction { + account: account_0, + perp_market, + owner, + side: Side::Bid, + price_lots, + max_base_lots: 10, + ..PerpPlaceOrderInstruction::default() + }, + ) + .await + .unwrap(); + send_tx( + solana, + PerpPlaceOrderInstruction { + account: account_1, + perp_market, + owner, + side: Side::Ask, + price_lots, + max_base_lots: 10, + ..PerpPlaceOrderInstruction::default() + }, + ) + .await + .unwrap(); + send_tx( + solana, + PerpConsumeEventsInstruction { + perp_market, + mango_accounts: vec![account_0, account_1], + }, + ) + .await + .unwrap(); + + // after this order exchange it is changed by + // 10*10*100*(0.5-1)*1.4 = -7000 for the long account0 + // 10*10*100*(1-1.5)*1.4 = -7000 for the short account1 + // (100 is base lot size) + assert_eq!( + account_init_health(solana, account_0).await.round(), + (10000.0f64 - 1000.5 * 1.4 - 7000.0).round() + ); + assert_eq!( + account_init_health(solana, account_1).await.round(), + 10000.0 - 7000.0 + ); + + // + // SETUP: Change the perp oracle to make perp-based health go positive for account_0 + // perp base value goes to 10*21*100*0.5, exceeding the negative quote + // perp uhupnl is 10*21*100*0.5 - 10*10*100 = 500 + // but health doesn't exceed 10k because of the 0 overall weight + // + set_perp_stub_oracle_price(solana, group, perp_market, &base_token, admin, 21.0).await; + assert_eq!( + account_init_health(solana, account_0).await.round(), + (10000.0f64 - 1000.5 * 1.4).round() + ); + + // + // SETUP: Increase the price of the borrow so account_0 becomes liquidatable + // + set_bank_stub_oracle_price(solana, group, &borrow_token, admin, 10.0).await; + assert_eq!( + account_init_health(solana, account_0).await.round(), + (10000.0f64 - 10.0 * 1000.5 * 1.4).round() + ); + + // + // SETUP: Fallback oracle + // + let fallback_oracle_kp = TestKeypair::new(); + let fallback_oracle = fallback_oracle_kp.pubkey(); + send_tx( + solana, + StubOracleCreate { + oracle: fallback_oracle_kp, + group, + mint: base_token.mint.pubkey, + admin, + payer, + }, + ) + .await + .unwrap(); + + send_tx( + solana, + PerpAddFallbackOracle { + group, + admin, + perp_market, + fallback_oracle, + }, + ) + .await + .unwrap(); + + send_tx( + solana, + TokenEdit { + group, + admin, + mint: base_token.mint.pubkey, + fallback_oracle, + options: mango_v4::instruction::TokenEdit { + set_fallback_oracle: true, + ..token_edit_instruction_default() + }, + }, + ) + .await + .unwrap(); + + // + // SETUP: Change the oracle to be invalid + // + send_tx( + solana, + StubOracleSetTestInstruction { + oracle: base_token.oracle, + group, + mint: base_token.mint.pubkey, + admin, + price: 21.0, + last_update_slot: 0, + deviation: 100.0, + }, + ) + .await + .unwrap(); + + // + // SETUP: Ensure fallback oracle matches default + // + send_tx( + solana, + StubOracleSetTestInstruction { + oracle: fallback_oracle, + group, + mint: base_token.mint.pubkey, + admin, + price: 21.0, + last_update_slot: 0, + deviation: 0.0, + }, + ) + .await + .unwrap(); + + // + // TEST: PerpLiqBaseOrPositivePnlInstruction fails with stale oracle + // + assert!(send_tx( + solana, + PerpLiqBaseOrPositivePnlInstruction { + liqor, + liqor_owner: owner, + liqee: account_0, + perp_market, + max_base_transfer: i64::MAX, + max_pnl_transfer: 100, + }, + ) + .await + .is_err()); + + // + // TEST: PerpLiqBaseOrPositivePnlInstruction succeeds with fallback + // + let fallback_oracle_meta = AccountMeta { + pubkey: fallback_oracle, + is_writable: false, + is_signer: false, + }; + assert!(send_tx_with_extra_accounts( + solana, + PerpLiqBaseOrPositivePnlInstruction { + liqor, + liqor_owner: owner, + liqee: account_0, + perp_market, + max_base_transfer: i64::MAX, + max_pnl_transfer: 100, + }, + vec![fallback_oracle_meta], + ) + .await + .unwrap() + .result + .is_ok()); + + let liqor_data = solana.get_account::(liqor).await; + assert_eq!(liqor_data.perps[0].base_position_lots(), 0); + assert_eq!(liqor_data.perps[0].quote_position_native(), 100); + assert_eq!( + account_position(solana, liqor, settle_token.bank).await, + 10000 - 95 + ); + let liqee_data = solana.get_account::(account_0).await; + assert_eq!(liqee_data.perps[0].base_position_lots(), 10); + assert_eq!(liqee_data.perps[0].quote_position_native(), -10100); + assert_eq!( + account_position(solana, account_0, settle_token.bank).await, + 95 + ); + + Ok(()) +} diff --git a/programs/mango-v4/tests/cases/test_perp.rs b/programs/mango-v4/tests/cases/test_perp.rs index 2b9fcfcd40..7f89ab648a 100644 --- a/programs/mango-v4/tests/cases/test_perp.rs +++ b/programs/mango-v4/tests/cases/test_perp.rs @@ -1713,7 +1713,7 @@ async fn test_perp_skip_bank() -> Result<(), TransportError> { Ok(()) } -async fn assert_no_perp_orders(solana: &SolanaCookie, account_0: Pubkey) { +pub async fn assert_no_perp_orders(solana: &SolanaCookie, account_0: Pubkey) { let mango_account_0 = solana.get_account::(account_0).await; for oo in mango_account_0.perp_open_orders.iter() { diff --git a/programs/mango-v4/tests/cases/test_stale_oracles.rs b/programs/mango-v4/tests/cases/test_stale_oracles.rs index 0dc51fbb71..caa0396f7b 100644 --- a/programs/mango-v4/tests/cases/test_stale_oracles.rs +++ b/programs/mango-v4/tests/cases/test_stale_oracles.rs @@ -1,6 +1,7 @@ use std::{path::PathBuf, str::FromStr}; use super::*; +use crate::cases::test_perp::assert_no_perp_orders; use anchor_lang::prelude::AccountMeta; use solana_sdk::account::AccountSharedData; @@ -743,3 +744,207 @@ async fn test_raydium_fallback_oracle() -> Result<(), TransportError> { Ok(()) } + +#[tokio::test] +async fn test_fallback_place_perp() -> Result<(), TransportError> { + let mut test_builder = TestContextBuilder::new(); + test_builder.test().set_compute_max_units(150_000); // bad oracles log a lot + let context = test_builder.start_default().await; + let solana = &context.solana.clone(); + + let admin = TestKeypair::new(); + let owner = context.users[0].key; + let payer = context.users[1].key; + let mints = &context.mints[0..2]; + + // + // SETUP: Create a group and an account + // + + let GroupWithTokens { group, tokens, .. } = GroupWithTokensConfig { + admin, + payer, + mints: mints.to_vec(), + ..GroupWithTokensConfig::default() + } + .create(solana) + .await; + + let deposit_amount = 1000; + let account_0 = create_funded_account( + &solana, + group, + owner, + 0, + &context.users[1], + mints, + deposit_amount, + 0, + ) + .await; + let account_1 = create_funded_account( + &solana, + group, + owner, + 1, + &context.users[1], + mints, + deposit_amount, + 0, + ) + .await; + let settler = + create_funded_account(&solana, group, owner, 251, &context.users[1], &[], 0, 0).await; + let settler_owner = owner.clone(); + + // + // TEST: Create a perp market + // + let mango_v4::accounts::PerpCreateMarket { + perp_market, bids, .. + } = send_tx( + solana, + PerpCreateMarketInstruction { + group, + admin, + payer, + perp_market_index: 0, + quote_lot_size: 10, + base_lot_size: 100, + maint_base_asset_weight: 0.975, + init_base_asset_weight: 0.95, + maint_base_liab_weight: 1.025, + init_base_liab_weight: 1.05, + base_liquidation_fee: 0.012, + maker_fee: -0.0001, + taker_fee: 0.0002, + settle_pnl_limit_factor: -1.0, + settle_pnl_limit_window_size_ts: 24 * 60 * 60, + ..PerpCreateMarketInstruction::with_new_book_and_queue(&solana, &tokens[0]).await + }, + ) + .await + .unwrap(); + + // + // SETUP: Fallback oracle + // + let fallback_oracle_kp = TestKeypair::new(); + let fallback_oracle = fallback_oracle_kp.pubkey(); + send_tx( + solana, + StubOracleCreate { + oracle: fallback_oracle_kp, + group, + mint: tokens[0].mint.pubkey, + admin, + payer, + }, + ) + .await + .unwrap(); + + send_tx( + solana, + PerpAddFallbackOracle { + group, + admin, + perp_market, + fallback_oracle, + }, + ) + .await + .unwrap(); + + let price_lots = { + let perp_market = solana.get_account::(perp_market).await; + perp_market.native_price_to_lot(I80F48::ONE) + }; + + // + // SETUP: Change the oracle to be invalid + // + send_tx( + solana, + StubOracleSetTestInstruction { + oracle: tokens[0].oracle, + group, + mint: tokens[0].mint.pubkey, + admin, + price: 1.0, + last_update_slot: 0, + deviation: 100.0, + }, + ) + .await + .unwrap(); + + // + // TEST: place order fails due to stale oracle + // + + let place_ix = PerpPlaceOrderInstruction { + account: account_0, + perp_market, + owner, + side: Side::Bid, + price_lots, + max_base_lots: 1, + ..PerpPlaceOrderInstruction::default() + }; + assert!(send_tx(solana, place_ix.clone()).await.is_err()); + + // + // SETUP: Ensure fallback oracle matches default + // + send_tx( + solana, + StubOracleSetTestInstruction { + oracle: tokens[0].oracle, + group, + mint: tokens[0].mint.pubkey, + admin, + price: 1.0, + last_update_slot: 0, + deviation: 0.0, + }, + ) + .await + .unwrap(); + + // + // TEST: place order succeeds with fallback oracle + // + let fallback_oracle_meta = AccountMeta { + pubkey: fallback_oracle, + is_writable: false, + is_signer: false, + }; + send_tx_with_extra_accounts(solana, place_ix, vec![fallback_oracle_meta.clone()]) + .await + .unwrap(); + + let bids_data = solana.get_account_boxed::(bids).await; + assert_eq!(bids_data.roots[0].leaf_count, 1); + let order_id_to_cancel = solana + .get_account::(account_0) + .await + .perp_open_orders[0] + .id; + + send_tx( + solana, + PerpCancelOrderInstruction { + account: account_0, + perp_market, + owner, + order_id: order_id_to_cancel, + }, + ) + .await + .unwrap(); + + assert_no_perp_orders(solana, account_0).await; + + Ok(()) +} diff --git a/programs/mango-v4/tests/program_test/mango_client.rs b/programs/mango-v4/tests/program_test/mango_client.rs index 2efa8967a4..a81e0b9b8f 100644 --- a/programs/mango-v4/tests/program_test/mango_client.rs +++ b/programs/mango-v4/tests/program_test/mango_client.rs @@ -3477,6 +3477,7 @@ fn perp_edit_instruction_default() -> mango_v4::instruction::PerpEditMarket { name_opt: None, force_close_opt: None, platform_liquidation_fee_opt: None, + set_fallback_oracle: false, } } @@ -3508,6 +3509,48 @@ impl ClientInstruction for PerpResetStablePriceModel { admin: self.admin.pubkey(), perp_market: self.perp_market, oracle: perp_market.oracle, + fallback_oracle: Pubkey::default(), + }; + + let instruction = make_instruction(program_id, &accounts, &instruction); + (accounts, instruction) + } + + fn signers(&self) -> Vec { + vec![self.admin] + } +} + +pub struct PerpAddFallbackOracle { + pub group: Pubkey, + pub admin: TestKeypair, + pub perp_market: Pubkey, + pub fallback_oracle: Pubkey, +} + +#[async_trait::async_trait(?Send)] +impl ClientInstruction for PerpAddFallbackOracle { + type Accounts = mango_v4::accounts::PerpEditMarket; + type Instruction = mango_v4::instruction::PerpEditMarket; + async fn to_instruction( + &self, + account_loader: &(impl ClientAccountLoader + 'async_trait), + ) -> (Self::Accounts, instruction::Instruction) { + let program_id = mango_v4::id(); + + let perp_market: PerpMarket = account_loader.load(&self.perp_market).await.unwrap(); + + let instruction = Self::Instruction { + set_fallback_oracle: true, + ..perp_edit_instruction_default() + }; + + let accounts = Self::Accounts { + group: self.group, + admin: self.admin.pubkey(), + perp_market: self.perp_market, + oracle: perp_market.oracle, + fallback_oracle: self.fallback_oracle, }; let instruction = make_instruction(program_id, &accounts, &instruction); @@ -3548,6 +3591,7 @@ impl ClientInstruction for PerpSetSettleLimitWindow { admin: self.admin.pubkey(), perp_market: self.perp_market, oracle: perp_market.oracle, + fallback_oracle: Pubkey::default(), }; let instruction = make_instruction(program_id, &accounts, &instruction); @@ -3590,6 +3634,7 @@ impl ClientInstruction for PerpMakeReduceOnly { admin: self.admin.pubkey(), perp_market: self.perp_market, oracle: perp_market.oracle, + fallback_oracle: Pubkey::default(), }; let instruction = make_instruction(program_id, &accounts, &instruction); @@ -3632,6 +3677,7 @@ impl ClientInstruction for PerpChangeWeights { admin: self.admin.pubkey(), perp_market: self.perp_market, oracle: perp_market.oracle, + fallback_oracle: Pubkey::default(), }; let instruction = make_instruction(program_id, &accounts, &instruction); @@ -3713,6 +3759,7 @@ impl ClientInstruction for PerpDeactivatePositionInstruction { } } +#[derive(Clone)] pub struct PerpPlaceOrderInstruction { pub account: Pubkey, pub perp_market: Pubkey, diff --git a/ts/client/scripts/liqtest/README.md b/ts/client/scripts/liqtest/README.md index 847fcfdaff..f1889404d3 100644 --- a/ts/client/scripts/liqtest/README.md +++ b/ts/client/scripts/liqtest/README.md @@ -51,6 +51,7 @@ This creates a bunch of to-be-liquidated accounts as well as a LIQOR account. Run the liquidator on the group with the liqor account. Since devnet doesn't have any jupiter, run with + ``` JUPITER_VERSION=mock TCS_MODE=borrow-buy diff --git a/ts/client/src/clientIxParamBuilder.ts b/ts/client/src/clientIxParamBuilder.ts index 3baa5a1d2d..ddfb1dd51f 100644 --- a/ts/client/src/clientIxParamBuilder.ts +++ b/ts/client/src/clientIxParamBuilder.ts @@ -311,6 +311,7 @@ export interface IxGateParams { Serum3PlaceOrderV2: boolean; TokenForceWithdraw: boolean; SequenceCheck: boolean; + HealthCheck: boolean; } // Default with all ixs enabled, use with buildIxGate @@ -392,6 +393,7 @@ export const TrueIxGateParams: IxGateParams = { Serum3PlaceOrderV2: true, TokenForceWithdraw: true, SequenceCheck: true, + HealthCheck: true, }; // build ix gate e.g. buildIxGate(Builder(TrueIxGateParams).TokenDeposit(false).build()).toNumber(), @@ -483,6 +485,7 @@ export function buildIxGate(p: IxGateParams): BN { toggleIx(ixGate, p, 'Serum3PlaceOrderV2', 71); toggleIx(ixGate, p, 'TokenForceWithdraw', 72); toggleIx(ixGate, p, 'SequenceCheck', 73); + toggleIx(ixGate, p, 'HealthCheck', 74); return ixGate; }