diff --git a/CHANGELOG.md b/CHANGELOG.md index dc093967ef..27957313f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,11 @@ incremented for features. * cli: Allow skipping the creation of a local validator when testing against localnet ([#93](https://github.com/project-serum/anchor/pull/93)). * cli: Adds support for tests with Typescript ([#94](https://github.com/project-serum/anchor/pull/94)). * cli: Deterministic and verifiable builds ([#100](https://github.com/project-serum/anchor/pull/100)). +* cli, lang: Add write buffers for IDL upgrades ([#107](https://github.com/project-serum/anchor/pull/107)). + +## Breaking Changes + +* lang: Removes `IdlInstruction::Clear` ([#107](https://github.com/project-serum/anchor/pull/107)). ## Fixes diff --git a/cli/Cargo.toml b/cli/Cargo.toml index fdc77f4644..9d28bcd799 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -8,6 +8,10 @@ edition = "2018" name = "anchor" path = "src/main.rs" +[features] +dev = [] +default = [] + [dependencies] clap = "3.0.0-beta.1" anyhow = "1.0.32" diff --git a/cli/src/main.rs b/cli/src/main.rs index 0dce58f478..4b6cb8c07b 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,7 +1,7 @@ //! CLI for workspace management of anchor programs. use crate::config::{read_all_programs, Config, Program}; -use anchor_lang::idl::IdlAccount; +use anchor_lang::idl::{IdlAccount, IdlInstruction}; use anchor_lang::{AccountDeserialize, AnchorDeserialize, AnchorSerialize}; use anchor_syn::idl::Idl; use anyhow::{anyhow, Result}; @@ -20,6 +20,7 @@ use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::Keypair; use solana_sdk::signature::Signer; +use solana_sdk::sysvar; use solana_sdk::transaction::Transaction; use std::fs::{self, File}; use std::io::prelude::*; @@ -110,6 +111,7 @@ pub enum Command { /// Filepath to the new program binary. program_filepath: String, }, + #[cfg(feature = "dev")] /// Runs an airdrop loop, continuously funding the configured wallet. Airdrop { #[clap(short, long)] @@ -125,7 +127,22 @@ pub enum IdlCommand { #[clap(short, long)] filepath: String, }, - /// Upgrades the IDL to the new file. + /// Writes an IDL into a buffer account. This can be used with SetBuffer + /// to perform an upgrade. + WriteBuffer { + program_id: Pubkey, + #[clap(short, long)] + filepath: String, + }, + /// Sets a new IDL buffer for the program. + SetBuffer { + program_id: Pubkey, + /// Address of the buffer account to set as the idl on the program. + #[clap(short, long)] + buffer: Pubkey, + }, + /// Upgrades the IDL to the new file. An alias for first writing and then + /// then setting the idl buffer account. Upgrade { program_id: Pubkey, #[clap(short, long)] @@ -133,6 +150,9 @@ pub enum IdlCommand { }, /// Sets a new authority on the IDL account. SetAuthority { + /// The IDL account buffer to set the authority of. If none is given, + /// then the canonical IDL account is used. + address: Option, /// Program to change the IDL authority. #[clap(short, long)] program_id: Pubkey, @@ -161,9 +181,10 @@ pub enum IdlCommand { #[clap(short, long)] out: Option, }, - /// Fetches an IDL for the given program from a cluster. + /// Fetches an IDL for the given address from a cluster. + /// The address can be a program, IDL account, or IDL buffer. Fetch { - program_id: Pubkey, + address: Pubkey, /// Output file for the idl (stdout if not specified). #[clap(short, long)] out: Option, @@ -193,6 +214,7 @@ fn main() -> Result<()> { skip_deploy, skip_local_validator, } => test(skip_deploy, skip_local_validator), + #[cfg(feature = "dev")] Command::Airdrop { url } => airdrop(url), } } @@ -531,17 +553,23 @@ fn verify_bin(program_id: Pubkey, bin_path: &Path, cluster: &str) -> Result Result { +fn fetch_idl(idl_addr: Pubkey) -> Result { let cfg = Config::discover()?.expect("Inside a workspace").0; let client = RpcClient::new(cfg.cluster.url().to_string()); - let idl_addr = IdlAccount::address(&program_id); - - let account = client + let mut account = client .get_account_with_commitment(&idl_addr, CommitmentConfig::processed())? .value .map_or(Err(anyhow!("Account not found")), Ok)?; + if account.executable { + let idl_addr = IdlAccount::address(&idl_addr); + account = client + .get_account_with_commitment(&idl_addr, CommitmentConfig::processed())? + .value + .map_or(Err(anyhow!("Account not found")), Ok)?; + } + // Cut off account discriminator. let mut d: &[u8] = &account.data[8..]; let idl_account: IdlAccount = AnchorDeserialize::deserialize(&mut d)?; @@ -563,18 +591,24 @@ fn idl(subcmd: IdlCommand) -> Result<()> { program_id, filepath, } => idl_init(program_id, filepath), + IdlCommand::WriteBuffer { + program_id, + filepath, + } => idl_write_buffer(program_id, filepath).map(|_| ()), + IdlCommand::SetBuffer { program_id, buffer } => idl_set_buffer(program_id, buffer), IdlCommand::Upgrade { program_id, filepath, } => idl_upgrade(program_id, filepath), IdlCommand::SetAuthority { program_id, + address, new_authority, - } => idl_set_authority(program_id, new_authority), + } => idl_set_authority(program_id, address, new_authority), IdlCommand::EraseAuthority { program_id } => idl_erase_authority(program_id), IdlCommand::Authority { program_id } => idl_authority(program_id), IdlCommand::Parse { file, out } => idl_parse(file, out), - IdlCommand::Fetch { program_id, out } => idl_fetch(program_id, out), + IdlCommand::Fetch { address, out } => idl_fetch(address, out), } } @@ -592,22 +626,86 @@ fn idl_init(program_id: Pubkey, idl_filepath: String) -> Result<()> { }) } -fn idl_upgrade(program_id: Pubkey, idl_filepath: String) -> Result<()> { +fn idl_write_buffer(program_id: Pubkey, idl_filepath: String) -> Result { with_workspace(|cfg, _path, _cargo| { + let keypair = cfg.wallet.to_string(); + let bytes = std::fs::read(idl_filepath)?; let idl: Idl = serde_json::from_reader(&*bytes)?; - idl_clear(cfg, &program_id)?; - idl_write(cfg, &program_id, &idl)?; + let idl_buffer = create_idl_buffer(&cfg, &keypair, &program_id, &idl)?; + idl_write(&cfg, &program_id, &idl, idl_buffer)?; + + println!("Idl buffer created: {:?}", idl_buffer); + + Ok(idl_buffer) + }) +} + +fn idl_set_buffer(program_id: Pubkey, buffer: Pubkey) -> Result<()> { + with_workspace(|cfg, _path, _cargo| { + let keypair = solana_sdk::signature::read_keypair_file(&cfg.wallet.to_string()) + .map_err(|_| anyhow!("Unable to read keypair file"))?; + let client = RpcClient::new(cfg.cluster.url().to_string()); + + // Instruction to set the buffer onto the IdlAccount. + let set_buffer_ix = { + let accounts = vec![ + AccountMeta::new(buffer, false), + AccountMeta::new(IdlAccount::address(&program_id), false), + AccountMeta::new(keypair.pubkey(), true), + ]; + let mut data = anchor_lang::idl::IDL_IX_TAG.to_le_bytes().to_vec(); + data.append(&mut IdlInstruction::SetBuffer.try_to_vec()?); + Instruction { + program_id, + accounts, + data, + } + }; + + // Build the transaction. + let (recent_hash, _fee_calc) = client.get_recent_blockhash()?; + let tx = Transaction::new_signed_with_payer( + &[set_buffer_ix], + Some(&keypair.pubkey()), + &[&keypair], + recent_hash, + ); + + // Send the transaction. + client.send_and_confirm_transaction_with_spinner_and_config( + &tx, + CommitmentConfig::confirmed(), + RpcSendTransactionConfig { + skip_preflight: true, + ..RpcSendTransactionConfig::default() + }, + )?; Ok(()) }) } +fn idl_upgrade(program_id: Pubkey, idl_filepath: String) -> Result<()> { + let buffer = idl_write_buffer(program_id, idl_filepath)?; + idl_set_buffer(program_id, buffer) +} + fn idl_authority(program_id: Pubkey) -> Result<()> { with_workspace(|cfg, _path, _cargo| { let client = RpcClient::new(cfg.cluster.url().to_string()); - let idl_address = IdlAccount::address(&program_id); + let idl_address = { + let account = client + .get_account_with_commitment(&program_id, CommitmentConfig::processed())? + .value + .map_or(Err(anyhow!("Account not found")), Ok)?; + if account.executable { + IdlAccount::address(&program_id) + } else { + program_id + } + }; let account = client.get_account(&idl_address)?; let mut data: &[u8] = &account.data; @@ -619,10 +717,17 @@ fn idl_authority(program_id: Pubkey) -> Result<()> { }) } -fn idl_set_authority(program_id: Pubkey, new_authority: Pubkey) -> Result<()> { +fn idl_set_authority( + program_id: Pubkey, + address: Option, + new_authority: Pubkey, +) -> Result<()> { with_workspace(|cfg, _path, _cargo| { // Misc. - let idl_address = IdlAccount::address(&program_id); + let idl_address = match address { + None => IdlAccount::address(&program_id), + Some(addr) => addr, + }; let keypair = solana_sdk::signature::read_keypair_file(&cfg.wallet.to_string()) .map_err(|_| anyhow!("Unable to read keypair file"))?; let client = RpcClient::new(cfg.cluster.url().to_string()); @@ -679,44 +784,7 @@ fn idl_erase_authority(program_id: Pubkey) -> Result<()> { // Program will treat the zero authority as erased. let new_authority = Pubkey::new_from_array([0u8; 32]); - idl_set_authority(program_id, new_authority)?; - - Ok(()) -} - -// Clears out *all* IDL data. The authority for the IDL must be the configured -// wallet. -fn idl_clear(cfg: &Config, program_id: &Pubkey) -> Result<()> { - let idl_address = IdlAccount::address(program_id); - let keypair = solana_sdk::signature::read_keypair_file(&cfg.wallet.to_string()) - .map_err(|_| anyhow!("Unable to read keypair file"))?; - let client = RpcClient::new(cfg.cluster.url().to_string()); - - let data = serialize_idl_ix(anchor_lang::idl::IdlInstruction::Clear)?; - let accounts = vec![ - AccountMeta::new(idl_address, false), - AccountMeta::new_readonly(keypair.pubkey(), true), - ]; - let ix = Instruction { - program_id: *program_id, - accounts, - data, - }; - let (recent_hash, _fee_calc) = client.get_recent_blockhash()?; - let tx = Transaction::new_signed_with_payer( - &[ix], - Some(&keypair.pubkey()), - &[&keypair], - recent_hash, - ); - client.send_and_confirm_transaction_with_spinner_and_config( - &tx, - CommitmentConfig::confirmed(), - RpcSendTransactionConfig { - skip_preflight: true, - ..RpcSendTransactionConfig::default() - }, - )?; + idl_set_authority(program_id, None, new_authority)?; Ok(()) } @@ -724,13 +792,12 @@ fn idl_clear(cfg: &Config, program_id: &Pubkey) -> Result<()> { // Write the idl to the account buffer, chopping up the IDL into pieces // and sending multiple transactions in the event the IDL doesn't fit into // a single transaction. -fn idl_write(cfg: &Config, program_id: &Pubkey, idl: &Idl) -> Result<()> { +fn idl_write(cfg: &Config, program_id: &Pubkey, idl: &Idl, idl_address: Pubkey) -> Result<()> { // Remove the metadata before deploy. let mut idl = idl.clone(); idl.metadata = None; // Misc. - let idl_address = IdlAccount::address(program_id); let keypair = solana_sdk::signature::read_keypair_file(&cfg.wallet.to_string()) .map_err(|_| anyhow!("Unable to read keypair file"))?; let client = RpcClient::new(cfg.cluster.url().to_string()); @@ -795,8 +862,8 @@ fn idl_parse(file: String, out: Option) -> Result<()> { write_idl(&idl, out) } -fn idl_fetch(program_id: Pubkey, out: Option) -> Result<()> { - let idl = fetch_idl(program_id)?; +fn idl_fetch(address: Pubkey, out: Option) -> Result<()> { + let idl = fetch_idl(address)?; let out = match out { None => OutFile::Stdout, Some(out) => OutFile::File(PathBuf::from(out)), @@ -1168,14 +1235,7 @@ fn create_idl_account( let keypair = solana_sdk::signature::read_keypair_file(keypair_path) .map_err(|_| anyhow!("Unable to read keypair file"))?; let client = RpcClient::new(cfg.cluster.url().to_string()); - - // Serialize and compress the idl. - let idl_data = { - let json_bytes = serde_json::to_vec(idl)?; - let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); - e.write_all(&json_bytes)?; - e.finish()? - }; + let idl_data = serialize_idl(idl)?; // Run `Create instruction. { @@ -1213,11 +1273,83 @@ fn create_idl_account( )?; } - idl_write(cfg, program_id, idl)?; + // Write directly to the IDL account buffer. + idl_write(cfg, program_id, idl, IdlAccount::address(program_id))?; Ok(idl_address) } +fn create_idl_buffer( + cfg: &Config, + keypair_path: &str, + program_id: &Pubkey, + idl: &Idl, +) -> Result { + let keypair = solana_sdk::signature::read_keypair_file(keypair_path) + .map_err(|_| anyhow!("Unable to read keypair file"))?; + let client = RpcClient::new(cfg.cluster.url().to_string()); + + let buffer = Keypair::generate(&mut OsRng); + + // Creates the new buffer account with the system program. + let create_account_ix = { + let space = 8 + 32 + 4 + serialize_idl(idl)?.len() as usize; + let lamports = client.get_minimum_balance_for_rent_exemption(space)?; + solana_sdk::system_instruction::create_account( + &keypair.pubkey(), + &buffer.pubkey(), + lamports, + space as u64, + program_id, + ) + }; + + // Program instruction to create the buffer. + let create_buffer_ix = { + let accounts = vec![ + AccountMeta::new(buffer.pubkey(), false), + AccountMeta::new_readonly(keypair.pubkey(), true), + AccountMeta::new_readonly(sysvar::rent::ID, false), + ]; + let mut data = anchor_lang::idl::IDL_IX_TAG.to_le_bytes().to_vec(); + data.append(&mut IdlInstruction::CreateBuffer.try_to_vec()?); + Instruction { + program_id: *program_id, + accounts, + data, + } + }; + + // Build the transaction. + let (recent_hash, _fee_calc) = client.get_recent_blockhash()?; + let tx = Transaction::new_signed_with_payer( + &[create_account_ix, create_buffer_ix], + Some(&keypair.pubkey()), + &[&keypair, &buffer], + recent_hash, + ); + + // Send the transaction. + client.send_and_confirm_transaction_with_spinner_and_config( + &tx, + CommitmentConfig::confirmed(), + RpcSendTransactionConfig { + skip_preflight: true, + ..RpcSendTransactionConfig::default() + }, + )?; + + Ok(buffer.pubkey()) +} + +// Serialize and compress the idl. +fn serialize_idl(idl: &Idl) -> Result> { + let json_bytes = serde_json::to_vec(idl)?; + let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(&json_bytes)?; + e.finish().map_err(Into::into) +} + fn serialize_idl_ix(ix_inner: anchor_lang::idl::IdlInstruction) -> Result> { let mut data = anchor_lang::idl::IDL_IX_TAG.to_le_bytes().to_vec(); data.append(&mut ix_inner.try_to_vec()?); @@ -1282,6 +1414,7 @@ fn set_workspace_dir_or_exit() { } } +#[cfg(feature = "dev")] fn airdrop(url: Option) -> Result<()> { let url = url.unwrap_or_else(|| "https://devnet.solana.com".to_string()); loop { diff --git a/lang/src/idl.rs b/lang/src/idl.rs index 2ab5c2fd92..0053bd0b98 100644 --- a/lang/src/idl.rs +++ b/lang/src/idl.rs @@ -27,10 +27,12 @@ pub const IDL_IX_TAG: u64 = 0x0a69e9a778bcf440; pub enum IdlInstruction { // One time initializer for creating the program's idl account. Create { data_len: u64 }, - // Appends to the end of the idl account data. + // Creates a new IDL account buffer. Can be called several times. + CreateBuffer, + // Appends the given data to the end of the idl account buffer. Write { data: Vec }, - // Clear's the IdlInstruction data. Used to update the IDL. - Clear, + // Sets a new data buffer for the IdlAccount. + SetBuffer, // Sets a new authority on the IdlAccount. SetAuthority { new_authority: Pubkey }, } @@ -47,8 +49,34 @@ pub struct IdlAccounts<'info> { pub authority: AccountInfo<'info>, } +// Accounts for creating an idl buffer. +#[derive(Accounts)] +pub struct IdlCreateBuffer<'info> { + #[account(init)] + pub buffer: ProgramAccount<'info, IdlAccount>, + #[account(signer, "authority.key != &Pubkey::new_from_array([0u8; 32])")] + pub authority: AccountInfo<'info>, + pub rent: Sysvar<'info, Rent>, +} + +// Accounts for upgrading the canonical IdlAccount with the buffer. +#[derive(Accounts)] +pub struct IdlSetBuffer<'info> { + // The buffer with the new idl data. + #[account(mut, "buffer.authority == idl.authority")] + pub buffer: ProgramAccount<'info, IdlAccount>, + // The idl account to be updated with the buffer's data. + #[account(mut, has_one = authority)] + pub idl: ProgramAccount<'info, IdlAccount>, + #[account(signer, "authority.key != &Pubkey::new_from_array([0u8; 32])")] + pub authority: AccountInfo<'info>, +} + // The account holding a program's IDL. This is stored on chain so that clients // can fetch it and generate a client with nothing but a program's ID. +// +// Note: we use the same account for the "write buffer", similar to the +// bpf upgradeable loader's mechanism. #[account] #[derive(Debug)] pub struct IdlAccount { diff --git a/lang/syn/src/codegen/program.rs b/lang/syn/src/codegen/program.rs index cf50afd42b..f834f81556 100644 --- a/lang/syn/src/codegen/program.rs +++ b/lang/syn/src/codegen/program.rs @@ -254,21 +254,26 @@ pub fn generate_non_inlined_handlers(program: &Program) -> proc_macro2::TokenStr __idl_create_account(program_id, &mut accounts, data_len)?; accounts.exit(program_id)?; }, - anchor_lang::idl::IdlInstruction::Write { data } => { - let mut accounts = anchor_lang::idl::IdlAccounts::try_accounts(program_id, &mut accounts)?; - __idl_write(program_id, &mut accounts, data)?; + anchor_lang::idl::IdlInstruction::CreateBuffer => { + let mut accounts = anchor_lang::idl::IdlCreateBuffer::try_accounts(program_id, &mut accounts)?; + __idl_create_buffer(program_id, &mut accounts)?; accounts.exit(program_id)?; }, - anchor_lang::idl::IdlInstruction::Clear => { + anchor_lang::idl::IdlInstruction::Write { data } => { let mut accounts = anchor_lang::idl::IdlAccounts::try_accounts(program_id, &mut accounts)?; - __idl_clear(program_id, &mut accounts)?; + __idl_write(program_id, &mut accounts, data)?; accounts.exit(program_id)?; }, anchor_lang::idl::IdlInstruction::SetAuthority { new_authority } => { let mut accounts = anchor_lang::idl::IdlAccounts::try_accounts(program_id, &mut accounts)?; __idl_set_authority(program_id, &mut accounts, new_authority)?; accounts.exit(program_id)?; - } + }, + anchor_lang::idl::IdlInstruction::SetBuffer => { + let mut accounts = anchor_lang::idl::IdlSetBuffer::try_accounts(program_id, &mut accounts)?; + __idl_set_buffer(program_id, &mut accounts)?; + accounts.exit(program_id)?; + }, } Ok(()) } @@ -338,6 +343,16 @@ pub fn generate_non_inlined_handlers(program: &Program) -> proc_macro2::TokenStr Ok(()) } + #[inline(never)] + pub fn __idl_create_buffer( + program_id: &Pubkey, + accounts: &mut anchor_lang::idl::IdlCreateBuffer, + ) -> ProgramResult { + let mut buffer = &mut accounts.buffer; + buffer.authority = *accounts.authority.key; + Ok(()) + } + #[inline(never)] pub fn __idl_write( program_id: &Pubkey, @@ -350,21 +365,21 @@ pub fn generate_non_inlined_handlers(program: &Program) -> proc_macro2::TokenStr } #[inline(never)] - pub fn __idl_clear( + pub fn __idl_set_authority( program_id: &Pubkey, accounts: &mut anchor_lang::idl::IdlAccounts, + new_authority: Pubkey, ) -> ProgramResult { - accounts.idl.data = vec![]; + accounts.idl.authority = new_authority; Ok(()) } #[inline(never)] - pub fn __idl_set_authority( + pub fn __idl_set_buffer( program_id: &Pubkey, - accounts: &mut anchor_lang::idl::IdlAccounts, - new_authority: Pubkey, + accounts: &mut anchor_lang::idl::IdlSetBuffer, ) -> ProgramResult { - accounts.idl.authority = new_authority; + accounts.idl.data = accounts.buffer.data.clone(); Ok(()) } } diff --git a/ts/package.json b/ts/package.json index db5742d78f..bd68baa211 100644 --- a/ts/package.json +++ b/ts/package.json @@ -1,6 +1,6 @@ { "name": "@project-serum/anchor", - "version": "0.2.2-beta.2", + "version": "0.2.2-beta.3", "description": "Anchor client", "main": "dist/cjs/index.js", "module": "dist/esm/index.js", diff --git a/ts/src/utils.ts b/ts/src/utils.ts index 4d0addb91d..e33319fda0 100644 --- a/ts/src/utils.ts +++ b/ts/src/utils.ts @@ -3,6 +3,7 @@ import { sha256 } from "crypto-hash"; import { struct } from "superstruct"; import assert from "assert"; import { PublicKey, AccountInfo, Connection } from "@solana/web3.js"; +import { idlAddress } from './idl'; export const TOKEN_PROGRAM_ID = new PublicKey( "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" @@ -112,6 +113,7 @@ const utils = { bs58, sha256, getMultipleAccounts, + idlAddress, }; export default utils;