IDL generation through compilation (coral-xyz#2011)

Co-authored-by: acheron <acheroncrypto@gmail.com>

.github/workflows/reusable-tests.yaml

@@ -443,6 +443,8 @@ jobs:
             path: tests/anchor-cli-account
           - cmd: cd tests/bench && anchor test --skip-lint
             path: tests/bench
+          - cmd: cd tests/idl-build && ./test.sh
+            path: tests/idl-build
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup/

CHANGELOG.md

@@ -15,6 +15,7 @@ The minor version will be incremented upon a breaking change and the patch versi
 - lang: Add `get_lamports`, `add_lamports` and `sub_lamports` methods for all account types ([#2552](https://github.com/coral-xyz/anchor/pull/2552)).
 - client: Add a helper struct `DynSigner` to simplify use of `Client<C> where <C: Clone + Deref<Target = impl Signer>>` with Solana clap CLI utils that loads `Signer` as `Box<dyn Signer>` ([#2550](https://github.com/coral-xyz/anchor/pull/2550)).
 - lang: Allow CPI calls matching an interface without pinning program ID ([#2559](https://github.com/coral-xyz/anchor/pull/2559)).
+- cli, lang: Add IDL generation through compilation. `anchor build` still uses parsing method to generate IDLs, use `anchor idl build` to generate IDLs with the build method ([#2011](https://github.com/coral-xyz/anchor/pull/2011)).
 
 ### Fixes
 
@@ -23,6 +24,8 @@ The minor version will be incremented upon a breaking change and the patch versi
 
 ### Breaking
 
+- syn: `idl` feature has been replaced with `idl-build`, `idl-parse` and `idl-types` features ([#2011](https://github.com/coral-xyz/anchor/pull/2011)).
+
 ## [0.28.0] - 2023-06-09
 
 ### Features

cli/Cargo.toml

@@ -19,7 +19,7 @@ default = []
 [dependencies]
 anchor-client = { path = "../client", version = "0.28.0" }
 anchor-lang = { path = "../lang", version = "0.28.0" }
-anchor-syn = { path = "../lang/syn", features = ["event-cpi", "idl", "init-if-needed"], version = "0.28.0" }
+anchor-syn = { path = "../lang/syn", features = ["event-cpi", "idl-parse", "init-if-needed"], version = "0.28.0" }
 anyhow = "1.0.32"
 base64 = "0.13.1"
 bincode = "1.3.3"

cli/src/config.rs

@@ -1,6 +1,6 @@
 use crate::is_hidden;
 use anchor_client::Cluster;
-use anchor_syn::idl::Idl;
+use anchor_syn::idl::types::Idl;
 use anyhow::{anyhow, bail, Context, Error, Result};
 use clap::{Parser, ValueEnum};
 use heck::ToSnakeCase;

cli/src/lib.rs

@@ -6,7 +6,10 @@ use crate::config::{
 use anchor_client::Cluster;
 use anchor_lang::idl::{IdlAccount, IdlInstruction, ERASED_AUTHORITY};
 use anchor_lang::{AccountDeserialize, AnchorDeserialize, AnchorSerialize};
-use anchor_syn::idl::{EnumFields, Idl, IdlType, IdlTypeDefinitionTy};
+use anchor_syn::idl::types::{
+    EnumFields, Idl, IdlConst, IdlErrorCode, IdlEvent, IdlType, IdlTypeDefinition,
+    IdlTypeDefinitionTy,
+};
 use anyhow::{anyhow, Context, Result};
 use clap::Parser;
 use flate2::read::GzDecoder;
@@ -417,6 +420,11 @@ pub enum IdlCommand {
         #[clap(long)]
         no_docs: bool,
     },
+    /// Generates the IDL for the program using the compilation method.
+    Build {
+        #[clap(long)]
+        no_docs: bool,
+    },
     /// Fetches an IDL for the given address from a cluster.
     /// The address can be a program, IDL account, or IDL buffer.
     Fetch {
@@ -1834,7 +1842,7 @@ fn extract_idl(
     let manifest_from_path = std::env::current_dir()?.join(PathBuf::from(&*file).parent().unwrap());
     let cargo = Manifest::discover_from_path(manifest_from_path)?
         .ok_or_else(|| anyhow!("Cargo.toml not found"))?;
-    anchor_syn::idl::file::parse(
+    anchor_syn::idl::parse::file::parse(
         &*file,
         cargo.version(),
         cfg.features.seeds,
@@ -1880,6 +1888,7 @@ fn idl(cfg_override: &ConfigOverride, subcmd: IdlCommand) -> Result<()> {
             out_ts,
             no_docs,
         } => idl_parse(cfg_override, file, out, out_ts, no_docs),
+        IdlCommand::Build { no_docs } => idl_build(no_docs),
         IdlCommand::Fetch { address, out } => idl_fetch(cfg_override, address, out),
     }
 }
@@ -2225,6 +2234,165 @@ fn idl_parse(
     Ok(())
 }
 
+fn idl_build(no_docs: bool) -> Result<()> {
+    let no_docs = if no_docs { "TRUE" } else { "FALSE" };
+
+    let cfg = Config::discover(&ConfigOverride::default())?.expect("Not in workspace.");
+    let seeds_feature = if cfg.features.seeds { "TRUE" } else { "FALSE" };
+
+    let exit = std::process::Command::new("cargo")
+        .args([
+            "test",
+            "__anchor_private_print_idl",
+            "--features",
+            "idl-build",
+            "--",
+            "--show-output",
+            "--quiet",
+        ])
+        .env("ANCHOR_IDL_GEN_NO_DOCS", no_docs)
+        .env("ANCHOR_IDL_GEN_SEEDS_FEATURE", seeds_feature)
+        .stderr(Stdio::inherit())
+        .output()
+        .map_err(|e| anyhow::format_err!("{}", e.to_string()))?;
+    if !exit.status.success() {
+        std::process::exit(exit.status.code().unwrap_or(1));
+    }
+
+    enum State {
+        Pass,
+        ConstLines(Vec<String>),
+        EventLines(Vec<String>),
+        ErrorsLines(Vec<String>),
+        ProgramLines(Vec<String>),
+    }
+
+    #[derive(Serialize, Deserialize)]
+    struct IdlGenEventPrint {
+        event: IdlEvent,
+        defined_types: Vec<IdlTypeDefinition>,
+    }
+
+    let mut state = State::Pass;
+
+    let mut events: Vec<IdlEvent> = vec![];
+    let mut error_codes: Option<Vec<IdlErrorCode>> = None;
+    let mut constants: Vec<IdlConst> = vec![];
+    let mut defined_types: BTreeMap<String, IdlTypeDefinition> = BTreeMap::new();
+    let mut curr_idl: Option<Idl> = None;
+
+    let mut idls: Vec<Idl> = vec![];
+
+    let out = String::from_utf8_lossy(&exit.stdout);
+    for line in out.lines() {
+        match &mut state {
+            State::Pass => {
+                if line == "---- IDL begin const ----" {
+                    state = State::ConstLines(vec![]);
+                    continue;
+                } else if line == "---- IDL begin event ----" {
+                    state = State::EventLines(vec![]);
+                    continue;
+                } else if line == "---- IDL begin errors ----" {
+                    state = State::ErrorsLines(vec![]);
+                    continue;
+                } else if line == "---- IDL begin program ----" {
+                    state = State::ProgramLines(vec![]);
+                    continue;
+                } else if line.starts_with("test result: ok") {
+                    let events = std::mem::take(&mut events);
+                    let error_codes = error_codes.take();
+                    let constants = std::mem::take(&mut constants);
+                    let mut defined_types = std::mem::take(&mut defined_types);
+                    let curr_idl = curr_idl.take();
+
+                    let events = if !events.is_empty() {
+                        Some(events)
+                    } else {
+                        None
+                    };
+
+                    let mut idl = match curr_idl {
+                        Some(idl) => idl,
+                        None => continue,
+                    };
+
+                    idl.events = events;
+                    idl.errors = error_codes;
+                    idl.constants = constants;
+
+                    idl.constants.sort_by(|a, b| a.name.cmp(&b.name));
+                    idl.accounts.sort_by(|a, b| a.name.cmp(&b.name));
+                    if let Some(e) = idl.events.as_mut() {
+                        e.sort_by(|a, b| a.name.cmp(&b.name))
+                    }
+
+                    let prog_ty = std::mem::take(&mut idl.types);
+                    defined_types
+                        .extend(prog_ty.into_iter().map(|ty| (ty.path.clone().unwrap(), ty)));
+                    idl.types = defined_types.into_values().collect::<Vec<_>>();
+
+                    idls.push(idl);
+                    continue;
+                }
+            }
+            State::ConstLines(lines) => {
+                if line == "---- IDL end const ----" {
+                    let constant: IdlConst = serde_json::from_str(&lines.join("\n"))?;
+                    constants.push(constant);
+                    state = State::Pass;
+                    continue;
+                }
+                lines.push(line.to_string());
+            }
+            State::EventLines(lines) => {
+                if line == "---- IDL end event ----" {
+                    let event: IdlGenEventPrint = serde_json::from_str(&lines.join("\n"))?;
+                    events.push(event.event);
+                    defined_types.extend(
+                        event
+                            .defined_types
+                            .into_iter()
+                            .map(|ty| (ty.path.clone().unwrap(), ty)),
+                    );
+                    state = State::Pass;
+                    continue;
+                }
+                lines.push(line.to_string());
+            }
+            State::ErrorsLines(lines) => {
+                if line == "---- IDL end errors ----" {
+                    let errs: Vec<IdlErrorCode> = serde_json::from_str(&lines.join("\n"))?;
+                    error_codes = Some(errs);
+                    state = State::Pass;
+                    continue;
+                }
+                lines.push(line.to_string());
+            }
+            State::ProgramLines(lines) => {
+                if line == "---- IDL end program ----" {
+                    let idl: Idl = serde_json::from_str(&lines.join("\n"))?;
+                    curr_idl = Some(idl);
+                    state = State::Pass;
+                    continue;
+                }
+                lines.push(line.to_string());
+            }
+        }
+    }
+
+    if idls.len() == 1 {
+        println!(
+            "{}",
+            serde_json::to_string_pretty(&idls.first().unwrap()).unwrap()
+        );
+    } else if idls.len() >= 2 {
+        println!("{}", serde_json::to_string_pretty(&idls).unwrap());
+    };
+
+    Ok(())
+}
+
 fn idl_fetch(cfg_override: &ConfigOverride, address: Pubkey, out: Option<String>) -> Result<()> {
     let idl = fetch_idl(cfg_override, address)?;
     let out = match out {
@@ -2500,6 +2668,11 @@ fn deserialize_idl_type_to_json(
 
             JsonValue::Array(array_data)
         }
+        IdlType::GenericLenArray(_, _) => todo!("Generic length arrays are not yet supported"),
+        IdlType::Generic(_) => todo!("Generic types are not yet supported"),
+        IdlType::DefinedWithTypeArgs { path: _, args: _ } => {
+            todo!("Defined types with type args are not yet supported")
+        }
     })
 }
 

cli/src/rust_template.rs

@@ -1,6 +1,6 @@
 use crate::config::ProgramWorkspace;
 use crate::VERSION;
-use anchor_syn::idl::Idl;
+use anchor_syn::idl::types::Idl;
 use anyhow::Result;
 use heck::{ToLowerCamelCase, ToSnakeCase, ToUpperCamelCase};
 use solana_sdk::{

cli/src/solidity_template.rs

@@ -1,6 +1,6 @@
 use crate::config::ProgramWorkspace;
 use crate::VERSION;
-use anchor_syn::idl::Idl;
+use anchor_syn::idl::types::Idl;
 use anyhow::Result;
 use heck::{ToLowerCamelCase, ToSnakeCase, ToUpperCamelCase};
 use solana_sdk::pubkey::Pubkey;

lang/Cargo.toml

@@ -14,6 +14,16 @@ init-if-needed = ["anchor-derive-accounts/init-if-needed"]
 derive = []
 default = []
 event-cpi = ["anchor-attribute-event/event-cpi"]
+idl-build = [
+    "anchor-syn/idl-build",
+    "anchor-derive-accounts/idl-build",
+    "anchor-derive-serde/idl-build",
+    "anchor-attribute-account/idl-build",
+    "anchor-attribute-constant/idl-build",
+    "anchor-attribute-event/idl-build",
+    "anchor-attribute-error/idl-build",
+    "anchor-attribute-program/idl-build",
+]
 anchor-debug = [
     "anchor-attribute-access-control/anchor-debug",
     "anchor-attribute-account/anchor-debug",
@@ -33,7 +43,11 @@ anchor-attribute-error = { path = "./attribute/error", version = "0.28.0" }
 anchor-attribute-event = { path = "./attribute/event", version = "0.28.0" }
 anchor-attribute-program = { path = "./attribute/program", version = "0.28.0" }
 anchor-derive-accounts = { path = "./derive/accounts", version = "0.28.0" }
+anchor-derive-serde = { path = "./derive/serde", version = "0.28.0" }
 anchor-derive-space = { path = "./derive/space", version = "0.28.0" }
+# anchor-syn can and should only be included only for idl-build. It won't compile
+# for bpf due to proc-macro2 crate.
+anchor-syn = { path = "./syn", version = "0.28.0", optional = true }
 arrayref = "0.3"
 base64 = "0.13"
 bincode = "1"

lang/attribute/account/Cargo.toml

@@ -12,6 +12,7 @@ edition = "2021"
 proc-macro = true
 
 [features]
+idl-build = ["anchor-syn/idl-build"]
 anchor-debug = ["anchor-syn/anchor-debug"]
 
 [dependencies]

lang/attribute/account/src/lib.rs

@@ -1,5 +1,7 @@
 extern crate proc_macro;
 
+#[cfg(feature = "idl-build")]
+use anchor_syn::idl::build::*;
 use quote::quote;
 use syn::parse_macro_input;
 
@@ -392,13 +394,26 @@ pub fn zero_copy(
         quote! {#[derive(::bytemuck::Zeroable)]}
     };
 
-    proc_macro::TokenStream::from(quote! {
+    let ret = quote! {
         #[derive(anchor_lang::__private::ZeroCopyAccessor, Copy, Clone)]
         #repr
         #pod
         #zeroable
         #account_strct
-    })
+    };
+
+    #[cfg(feature = "idl-build")]
+    {
+        let no_docs = get_no_docs();
+        let idl_gen_impl = gen_idl_gen_impl_for_struct(&account_strct, no_docs);
+        return proc_macro::TokenStream::from(quote! {
+            #ret
+            #idl_gen_impl
+        });
+    }
+
+    #[allow(unreachable_code)]
+    proc_macro::TokenStream::from(ret)
 }
 
 /// Defines the program's ID. This should be used at the root of all Anchor

lang/attribute/constant/Cargo.toml

@@ -12,9 +12,11 @@ edition = "2021"
 proc-macro = true
 
 [features]
+idl-build = ["anchor-syn/idl-build"]
 anchor-debug = ["anchor-syn/anchor-debug"]
 
 [dependencies]
 anchor-syn = { path = "../../syn", version = "0.28.0" }
 proc-macro2 = "1"
+quote = "1"
 syn = { version = "1", features = ["full"] }