From 5845e92f109974edce754179db541e373dc532a9 Mon Sep 17 00:00:00 2001 From: maciektr Date: Fri, 15 Nov 2024 11:38:44 +0100 Subject: [PATCH] Allocate token's content into an arena (#1722) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit-id:5e4b122f Add token stream ser/de tests --- **Stack**: - #1749 - #1748 - #1745 - #1747 - #1722 ⬅ ⚠️ *Part of a stack created by [spr](https://github.com/ejoffe/spr). Do not merge manually using the UI - doing so may have unexpected results.* --- Cargo.lock | 143 +++++---- Cargo.toml | 49 ++-- plugins/cairo-lang-macro-stable/src/lib.rs | 4 +- plugins/cairo-lang-macro/Cargo.toml | 3 +- plugins/cairo-lang-macro/src/lib.rs | 64 ++-- .../cairo-lang-macro/src/types/conversion.rs | 39 ++- plugins/cairo-lang-macro/src/types/mod.rs | 2 +- plugins/cairo-lang-macro/src/types/token.rs | 276 +++++++++++++++++- scarb/src/compiler/plugin/proc_macro/host.rs | 75 +++-- scarb/src/compiler/plugin/proc_macro/types.rs | 41 +-- scarb/tests/build_cairo_plugin.rs | 11 +- scarb/tests/package.rs | 1 + scarb/tests/proc_macro_server.rs | 6 +- 13 files changed, 505 insertions(+), 209 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3dd4c003c..46abfd3f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -562,8 +562,8 @@ dependencies = [ [[package]] name = "cairo-lang-casm" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-utils", "indoc", @@ -575,8 +575,8 @@ dependencies = [ [[package]] name = "cairo-lang-compiler" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "anyhow", "cairo-lang-defs", @@ -600,16 +600,16 @@ dependencies = [ [[package]] name = "cairo-lang-debug" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-utils", ] [[package]] name = "cairo-lang-defs" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-debug", "cairo-lang-diagnostics", @@ -624,8 +624,8 @@ dependencies = [ [[package]] name = "cairo-lang-diagnostics" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-debug", "cairo-lang-filesystem", @@ -635,8 +635,8 @@ dependencies = [ [[package]] name = "cairo-lang-doc" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-debug", "cairo-lang-defs", @@ -655,8 +655,8 @@ dependencies = [ [[package]] name = "cairo-lang-eq-solver" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-utils", "good_lp", @@ -664,8 +664,8 @@ dependencies = [ [[package]] name = "cairo-lang-filesystem" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-debug", "cairo-lang-utils", @@ -679,8 +679,8 @@ dependencies = [ [[package]] name = "cairo-lang-formatter" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "anyhow", "cairo-lang-diagnostics", @@ -698,8 +698,8 @@ dependencies = [ [[package]] name = "cairo-lang-language-server" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "anyhow", "cairo-lang-compiler", @@ -738,8 +738,8 @@ dependencies = [ [[package]] name = "cairo-lang-lowering" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-debug", "cairo-lang-defs", @@ -764,7 +764,8 @@ dependencies = [ name = "cairo-lang-macro" version = "0.1.1" dependencies = [ - "cairo-lang-macro-attributes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bumpalo", + "cairo-lang-macro-attributes", "cairo-lang-macro-stable", "linkme", "serde", @@ -780,25 +781,14 @@ dependencies = [ "syn 2.0.87", ] -[[package]] -name = "cairo-lang-macro-attributes" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e32e958decd95ae122ee64daa26721da2f76e83231047f947fd9cdc5d3c90cc6" -dependencies = [ - "quote", - "scarb-stable-hash 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 2.0.87", -] - [[package]] name = "cairo-lang-macro-stable" version = "1.0.0" [[package]] name = "cairo-lang-parser" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-diagnostics", "cairo-lang-filesystem", @@ -816,8 +806,8 @@ dependencies = [ [[package]] name = "cairo-lang-plugins" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-defs", "cairo-lang-diagnostics", @@ -834,8 +824,8 @@ dependencies = [ [[package]] name = "cairo-lang-proc-macros" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-debug", "quote", @@ -844,8 +834,8 @@ dependencies = [ [[package]] name = "cairo-lang-project" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-filesystem", "cairo-lang-utils", @@ -856,8 +846,8 @@ dependencies = [ [[package]] name = "cairo-lang-runnable-utils" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-casm", "cairo-lang-sierra", @@ -867,13 +857,14 @@ dependencies = [ "cairo-lang-sierra-type-size", "cairo-lang-utils", "cairo-vm", + "itertools 0.12.1", "thiserror 1.0.69", ] [[package]] name = "cairo-lang-runner" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "ark-ff", "ark-secp256k1", @@ -901,8 +892,8 @@ dependencies = [ [[package]] name = "cairo-lang-semantic" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-debug", "cairo-lang-defs", @@ -926,8 +917,8 @@ dependencies = [ [[package]] name = "cairo-lang-sierra" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "anyhow", "cairo-lang-utils", @@ -952,8 +943,8 @@ dependencies = [ [[package]] name = "cairo-lang-sierra-ap-change" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-eq-solver", "cairo-lang-sierra", @@ -967,8 +958,8 @@ dependencies = [ [[package]] name = "cairo-lang-sierra-gas" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-eq-solver", "cairo-lang-sierra", @@ -982,8 +973,8 @@ dependencies = [ [[package]] name = "cairo-lang-sierra-generator" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-debug", "cairo-lang-defs", @@ -1005,8 +996,8 @@ dependencies = [ [[package]] name = "cairo-lang-sierra-to-casm" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "assert_matches", "cairo-lang-casm", @@ -1025,8 +1016,8 @@ dependencies = [ [[package]] name = "cairo-lang-sierra-type-size" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-sierra", "cairo-lang-utils", @@ -1034,8 +1025,8 @@ dependencies = [ [[package]] name = "cairo-lang-starknet" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "anyhow", "cairo-lang-compiler", @@ -1063,8 +1054,8 @@ dependencies = [ [[package]] name = "cairo-lang-starknet-classes" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-casm", "cairo-lang-sierra", @@ -1085,8 +1076,8 @@ dependencies = [ [[package]] name = "cairo-lang-syntax" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-debug", "cairo-lang-filesystem", @@ -1100,8 +1091,8 @@ dependencies = [ [[package]] name = "cairo-lang-syntax-codegen" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "genco", "xshell", @@ -1109,8 +1100,8 @@ dependencies = [ [[package]] name = "cairo-lang-test-plugin" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "anyhow", "cairo-lang-compiler", @@ -1135,8 +1126,8 @@ dependencies = [ [[package]] name = "cairo-lang-test-runner" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "anyhow", "cairo-lang-compiler", @@ -1157,8 +1148,8 @@ dependencies = [ [[package]] name = "cairo-lang-test-utils" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "cairo-lang-formatter", "cairo-lang-utils", @@ -1169,8 +1160,8 @@ dependencies = [ [[package]] name = "cairo-lang-utils" -version = "2.8.5" -source = "git+https://github.com/starkware-libs/cairo?rev=6aa7c753e1d509d7015c0027b908bf8d77a5e200#6aa7c753e1d509d7015c0027b908bf8d77a5e200" +version = "2.8.4" +source = "git+https://github.com/starkware-libs/cairo?rev=3874879adcc2f36d7e1235fc9aec8420ff7013b0#3874879adcc2f36d7e1235fc9aec8420ff7013b0" dependencies = [ "env_logger", "hashbrown 0.14.5", diff --git a/Cargo.toml b/Cargo.toml index 7b4c12899..e44ca83de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,30 +35,31 @@ anyhow = "1" assert_fs = "1" async-trait = "0.1" axum = { version = "0.6", features = ["http2"] } -cairo-lang-casm = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-compiler = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-debug = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-defs = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-diagnostics = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-doc = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-filesystem = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-formatter = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-language-server = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-lowering = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-parser = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-project = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-runner = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-semantic = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-sierra = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-sierra-generator = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-sierra-to-casm = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-sierra-type-size = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-starknet = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-starknet-classes = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-syntax = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-test-plugin = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-test-runner = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200" } -cairo-lang-utils = { git = "https://github.com/starkware-libs/cairo", rev = "6aa7c753e1d509d7015c0027b908bf8d77a5e200", features = ["env_logger"] } +bumpalo = "3" +cairo-lang-casm = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-compiler = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-debug = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-defs = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-diagnostics = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-doc = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-filesystem = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-formatter = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-language-server = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-lowering = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-parser = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-project = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-runner = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-semantic = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-sierra = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-sierra-generator = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-sierra-to-casm = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-sierra-type-size = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-starknet = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-starknet-classes = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-syntax = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-test-plugin = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-test-runner = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" } +cairo-lang-utils = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0", features = ["env_logger"] } camino = { version = "1", features = ["serde1"] } cargo_metadata = ">=0.18" clap = { version = "4", features = ["derive", "env", "string"] } diff --git a/plugins/cairo-lang-macro-stable/src/lib.rs b/plugins/cairo-lang-macro-stable/src/lib.rs index 912a72847..c836ed850 100644 --- a/plugins/cairo-lang-macro-stable/src/lib.rs +++ b/plugins/cairo-lang-macro-stable/src/lib.rs @@ -9,7 +9,8 @@ pub mod ffi; #[derive(Debug)] pub struct StableToken { pub span: StableTextSpan, - pub content: *mut c_char, + pub ptr: *const u8, + pub len: usize, } #[repr(C)] @@ -44,6 +45,7 @@ pub type StableExpansionsList = StableSlice; pub struct StableTokenStream { pub tokens: StableSlice, pub metadata: StableTokenStreamMetadata, + pub size_hint: usize, } /// Token stream metadata. diff --git a/plugins/cairo-lang-macro/Cargo.toml b/plugins/cairo-lang-macro/Cargo.toml index 55d274eb5..659c9d0e1 100644 --- a/plugins/cairo-lang-macro/Cargo.toml +++ b/plugins/cairo-lang-macro/Cargo.toml @@ -13,7 +13,8 @@ readme = "README.md" repository.workspace = true [dependencies] -cairo-lang-macro-attributes = "0.1" +bumpalo.workspace = true +cairo-lang-macro-attributes = { path = "../cairo-lang-macro-attributes" } cairo-lang-macro-stable = { path = "../cairo-lang-macro-stable" } linkme.workspace = true serde = { workspace = true, optional = true } diff --git a/plugins/cairo-lang-macro/src/lib.rs b/plugins/cairo-lang-macro/src/lib.rs index 0eb6bd16b..4b31e04ad 100644 --- a/plugins/cairo-lang-macro/src/lib.rs +++ b/plugins/cairo-lang-macro/src/lib.rs @@ -18,17 +18,22 @@ pub use cairo_lang_macro_attributes::*; #[doc(hidden)] pub use linkme; +use std::cell::RefCell; use cairo_lang_macro_stable::ffi::StableSlice; use cairo_lang_macro_stable::{ StableExpansionsList, StablePostProcessContext, StableProcMacroResult, }; use std::ffi::{c_char, CStr, CString}; +use std::ops::Deref; mod types; pub use types::*; +// A thread-local allocation context for allocating tokens on proc macro side. +thread_local!(static CONTEXT: RefCell = RefCell::default() ); + #[doc(hidden)] #[derive(Clone)] pub struct ExpansionDefinition { @@ -97,29 +102,42 @@ pub unsafe extern "C" fn expand( stable_attr: cairo_lang_macro_stable::StableTokenStream, stable_token_stream: cairo_lang_macro_stable::StableTokenStream, ) -> cairo_lang_macro_stable::StableResultWrapper { - let token_stream = TokenStream::from_stable(&stable_token_stream); - let attr_token_stream = TokenStream::from_stable(&stable_attr); - let item_name = CStr::from_ptr(item_name).to_string_lossy().to_string(); - let fun = MACRO_DEFINITIONS_SLICE - .iter() - .find_map(|m| { - if m.name == item_name.as_str() { - Some(m.fun.clone()) - } else { - None - } - }) - .expect("procedural macro not found"); - let result = match fun { - ExpansionFunc::Attr(fun) => fun(attr_token_stream, token_stream), - ExpansionFunc::Other(fun) => fun(token_stream), - }; - let result: StableProcMacroResult = result.into_stable(); - cairo_lang_macro_stable::StableResultWrapper { - input: stable_token_stream, - input_attr: stable_attr, - output: result, - } + CONTEXT.with(|ctx_cell| { + // Read size hint from stable token stream. This will be used to create a sufficiently + // large bump allocation buffer. + let size_hint: usize = stable_token_stream.size_hint + stable_attr.size_hint; + // Replace the allocation context with a new one. + // If there is no interned string guards, the old context will be de-allocated. + ctx_cell.replace(AllocationContext::with_capacity(size_hint)); + let ctx_borrow = ctx_cell.borrow(); + let ctx: &AllocationContext = ctx_borrow.deref(); + // Copy the stable token stream into current context. + let token_stream = TokenStream::from_stable_in(&stable_token_stream, ctx); + let attr_token_stream = TokenStream::from_stable_in(&stable_attr, ctx); + let item_name = CStr::from_ptr(item_name) + .to_str() + .expect("item name must be a valid string"); + let fun = MACRO_DEFINITIONS_SLICE + .iter() + .find_map(|m| { + if m.name == item_name { + Some(m.fun.clone()) + } else { + None + } + }) + .expect("procedural macro not found"); + let result = match fun { + ExpansionFunc::Attr(fun) => fun(attr_token_stream, token_stream), + ExpansionFunc::Other(fun) => fun(token_stream), + }; + let result: StableProcMacroResult = result.into_stable(); + cairo_lang_macro_stable::StableResultWrapper { + input: stable_token_stream, + input_attr: stable_attr, + output: result, + } + }) } /// Free the memory allocated for the [`StableProcMacroResult`]. diff --git a/plugins/cairo-lang-macro/src/types/conversion.rs b/plugins/cairo-lang-macro/src/types/conversion.rs index 924ee4c56..229381e0b 100644 --- a/plugins/cairo-lang-macro/src/types/conversion.rs +++ b/plugins/cairo-lang-macro/src/types/conversion.rs @@ -1,6 +1,7 @@ use crate::{ - AuxData, Diagnostic, ExpansionDefinition, FullPathMarker, PostProcessContext, ProcMacroResult, - Severity, TextSpan, Token, TokenStream, TokenStreamMetadata, TokenTree, + AllocationContext, AuxData, Diagnostic, ExpansionDefinition, FullPathMarker, + PostProcessContext, ProcMacroResult, Severity, TextSpan, Token, TokenStream, + TokenStreamMetadata, TokenTree, }; use cairo_lang_macro_stable::ffi::StableSlice; use cairo_lang_macro_stable::{ @@ -44,6 +45,7 @@ impl ProcMacroResult { /// # Safety #[doc(hidden)] pub unsafe fn from_stable(result: &StableProcMacroResult) -> Self { + let ctx = AllocationContext::with_capacity(result.token_stream.size_hint); let (ptr, n) = result.diagnostics.raw_parts(); let diagnostics = slice::from_raw_parts(ptr, n) .iter() @@ -55,7 +57,7 @@ impl ProcMacroResult { .map(|m| from_raw_cstr(*m)) .collect::>(); ProcMacroResult { - token_stream: TokenStream::from_stable(&result.token_stream), + token_stream: TokenStream::from_stable_in(&result.token_stream, &ctx), diagnostics, full_path_markers, aux_data: AuxData::from_stable(&result.aux_data), @@ -110,10 +112,12 @@ impl Token { /// Convert to FFI-safe representation. #[doc(hidden)] pub fn into_stable(self) -> StableToken { - let cstr = CString::new(self.content.as_bytes()).unwrap(); + let ptr = self.content.as_ptr(); + let len = self.content.len(); StableToken { span: self.span.into_stable(), - content: cstr.into_raw(), + ptr, + len, } } @@ -123,9 +127,11 @@ impl Token { /// /// # Safety #[doc(hidden)] - pub unsafe fn from_stable(token: &StableToken) -> Self { + pub unsafe fn from_stable_in(token: &StableToken, ctx: &AllocationContext) -> Self { + let content = slice::from_raw_parts(token.ptr, token.len); + let content = ctx.intern(std::str::from_utf8_unchecked(content)); Self { - content: from_raw_cstr(token.content), + content, span: TextSpan::from_stable(&token.span), } } @@ -138,7 +144,6 @@ impl Token { /// # Safety #[doc(hidden)] pub unsafe fn free_owned_stable(token: StableToken) { - free_raw_cstring(token.content); TextSpan::free_owned_stable(token.span); } } @@ -158,9 +163,9 @@ impl TokenTree { /// /// # Safety #[doc(hidden)] - pub unsafe fn from_stable(token_tree: &StableTokenTree) -> Self { + pub unsafe fn from_stable_in(token_tree: &StableTokenTree, ctx: &AllocationContext) -> Self { match token_tree { - StableTokenTree::Ident(token) => Self::Ident(Token::from_stable(token)), + StableTokenTree::Ident(token) => Self::Ident(Token::from_stable_in(token, ctx)), } } @@ -186,14 +191,19 @@ impl TokenStream { /// # Safety #[doc(hidden)] pub fn into_stable(self) -> StableTokenStream { + let mut size_hint: usize = 0; let tokens = self .tokens .into_iter() - .map(|token| token.into_stable()) + .map(|token| { + size_hint += token.size_hint(); + token.into_stable() + }) .collect::>(); StableTokenStream { tokens: StableSlice::new(tokens), metadata: self.metadata.into_stable(), + size_hint, } } @@ -203,11 +213,14 @@ impl TokenStream { /// /// # Safety #[doc(hidden)] - pub unsafe fn from_stable(token_stream: &StableTokenStream) -> Self { + pub unsafe fn from_stable_in( + token_stream: &StableTokenStream, + ctx: &AllocationContext, + ) -> Self { let (ptr, n) = token_stream.tokens.raw_parts(); let tokens = slice::from_raw_parts(ptr, n) .iter() - .map(|token_tree| TokenTree::from_stable(token_tree)) + .map(|token_tree| TokenTree::from_stable_in(token_tree, ctx)) .collect::>(); Self { tokens, diff --git a/plugins/cairo-lang-macro/src/types/mod.rs b/plugins/cairo-lang-macro/src/types/mod.rs index 618d61746..45ab5fca2 100644 --- a/plugins/cairo-lang-macro/src/types/mod.rs +++ b/plugins/cairo-lang-macro/src/types/mod.rs @@ -44,7 +44,7 @@ pub struct ProcMacroResult { /// let token_stream = TokenStream::new(vec![ /// TokenTree::Ident( /// Token::new( -/// code.clone(), +/// &code, /// TextSpan::new(0, code.len()) /// ) /// ) diff --git a/plugins/cairo-lang-macro/src/types/token.rs b/plugins/cairo-lang-macro/src/types/token.rs index 97f622fa1..734b94e56 100644 --- a/plugins/cairo-lang-macro/src/types/token.rs +++ b/plugins/cairo-lang-macro/src/types/token.rs @@ -1,17 +1,85 @@ +use crate::CONTEXT; +use bumpalo::Bump; use std::fmt::Display; +use std::hash::{Hash, Hasher}; +use std::ops::Deref; +use std::rc::Rc; /// An abstract stream of Cairo tokens. /// /// This is both input and part of an output of a procedural macro. #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(try_from = "deserializer::TokenStream"))] #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] pub struct TokenStream { pub tokens: Vec, pub metadata: TokenStreamMetadata, } +/// This module implements deserialization of the token stream, for the serde feature. +/// This is intermediate representation is needed, as real [`Token`] only contains a reference to the +/// represented string, which needs to be allocated outside the [`Token`] struct. +/// Here we allocate each token to an owned String with SerDe and then copy it's content into context. +#[cfg(feature = "serde")] +#[doc(hidden)] +mod deserializer { + use crate::{AllocationContext, TextSpan, TokenStreamMetadata}; + use std::fmt::{Display, Formatter}; + + #[derive(serde::Serialize, serde::Deserialize)] + pub struct TokenStream { + pub tokens: Vec, + pub metadata: TokenStreamMetadata, + } + + #[derive(serde::Serialize, serde::Deserialize)] + pub enum TokenTree { + Ident(Token), + } + + #[derive(serde::Serialize, serde::Deserialize)] + pub struct Token { + pub content: String, + pub span: TextSpan, + } + + pub struct Error {} + + impl Display for Error { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str("TokenStream deserialization error") + } + } + + impl TryFrom for crate::TokenStream { + type Error = Error; + + fn try_from(value: TokenStream) -> Result { + let ctx = AllocationContext::default(); + let tokens = value + .tokens + .into_iter() + .map(|token| match token { + TokenTree::Ident(token) => { + let content = ctx.intern(token.content.as_str()); + let token = crate::Token { + content, + span: token.span, + }; + crate::TokenTree::Ident(token) + } + }) + .collect::>(); + Ok(Self { + tokens, + metadata: value.metadata, + }) + } + } +} + /// A single token or a delimited sequence of token trees. -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum TokenTree { Ident(Token), @@ -23,6 +91,16 @@ impl Default for TokenTree { } } +impl TokenTree { + /// Get the size hint for the [`TokenTree`]. + /// This can be used to estimate size of a buffer needed for allocating this [`TokenTree`]. + pub(crate) fn size_hint(&self) -> usize { + match self { + Self::Ident(token) => token.size_hint(), + } + } +} + /// A range of text offsets that form a span (like text selection). #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] @@ -34,13 +112,129 @@ pub struct TextSpan { /// A single Cairo token. /// /// The most atomic item of Cairo code representation. -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] pub struct Token { - pub content: String, + pub content: InternedStr, pub span: TextSpan, } +impl Token { + /// Get the size hint for the [`Token`]. + /// This can be used to estimate size of a buffer needed for allocating this [`Token`]. + pub(crate) fn size_hint(&self) -> usize { + self.content.deref().len() + } +} + +/// A wrapper over a string pointer. +/// This contains a pointer to a string allocated in a bump allocator +/// and a guard which keeps the buffer alive. +/// This way we do not need to allocate a new string, +/// but also do not need to worry about the lifetime of the string. +#[derive(Debug, Clone)] +pub struct InternedStr { + ptr: *const str, + // Holding a rc to the underlying buffer, so that ptr will always point to valid memory. + _bump: Rc, +} + +impl InternedStr { + #[allow(private_interfaces)] + #[doc(hidden)] + pub(crate) fn new_in(s: &str, bump: Rc) -> Self { + let allocated = bump.0.alloc_str(s); + let ptr = allocated as *const str; + Self { ptr, _bump: bump } + } +} + +impl Default for InternedStr { + fn default() -> Self { + Self { + ptr: "" as *const str, + _bump: Rc::default(), + } + } +} + +impl AsRef for InternedStr { + fn as_ref(&self) -> &str { + self.deref() + } +} + +impl Deref for InternedStr { + type Target = str; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.ptr } + } +} + +#[cfg(feature = "serde")] +impl serde::Serialize for InternedStr { + fn serialize(&self, s: S) -> Result { + s.serialize_str(self.as_ref()) + } +} + +impl PartialEq for InternedStr { + fn eq(&self, other: &Self) -> bool { + self.as_ref().eq(other.as_ref()) + } +} + +impl Eq for InternedStr {} + +impl Hash for InternedStr { + fn hash(&self, state: &mut H) { + self.as_ref().hash(state); + } +} + +/// This wrapper de-allocates the underlying buffer on drop. +#[derive(Debug, Default)] +struct BumpWrap(pub Bump); + +impl Drop for BumpWrap { + fn drop(&mut self) { + self.0.reset(); + } +} + +/// A context for allocating Cairo tokens. +/// This wrapper contains a bump allocator, which is used to allocate strings for tokens. +#[derive(Clone)] +pub struct AllocationContext { + bump: Rc, +} + +impl AllocationContext { + /// Allocate a new context with pre-determined buffer size. + pub fn with_capacity(size_hint: usize) -> Self { + Self { + bump: Rc::new(BumpWrap(Bump::with_capacity(size_hint))), + } + } + + /// Allocate a string in the context. + /// This returned a string pointer, guarded by reference counter to the buffer. + /// The buffer will be deallocated when the last reference to the buffer is dropped. + /// No special handling or lifetimes are needed for the string. + pub(crate) fn intern(&self, value: &str) -> InternedStr { + InternedStr::new_in(value, self.bump.clone()) + } +} + +impl Default for AllocationContext { + fn default() -> Self { + Self { + bump: Rc::new(BumpWrap(Bump::new())), + } + } +} + /// Metadata of [`TokenStream`]. /// /// This struct describes the origin of the [`TokenStream`]. @@ -84,8 +278,9 @@ impl TokenStream { &self.metadata } + /// Check if the [`TokenStream`] is empty. pub fn is_empty(&self) -> bool { - self.to_string().is_empty() + self.tokens.is_empty() } } @@ -94,7 +289,7 @@ impl Display for TokenStream { for token in &self.tokens { match token { TokenTree::Ident(token) => { - write!(f, "{}", token.content.clone())?; + write!(f, "{}", token.content.as_ref())?; } } } @@ -114,19 +309,88 @@ impl TokenStreamMetadata { } impl TokenTree { + /// Create a new [`TokenTree`] from an identifier [`Token`]. pub fn from_ident(token: Token) -> Self { Self::Ident(token) } } impl TextSpan { + /// Create a new [`TextSpan`]. pub fn new(start: usize, end: usize) -> TextSpan { TextSpan { start, end } } } impl Token { - pub fn new(content: String, span: TextSpan) -> Self { + /// Create [`Token`] in thread-local context. + pub fn new(content: impl AsRef, span: TextSpan) -> Self { + CONTEXT.with(|ctx| { + let ctx_borrow = ctx.borrow(); + let ctx: &AllocationContext = ctx_borrow.deref(); + Self::new_in(content, span, ctx) + }) + } + + /// Create [`Token`] in specified context. + pub fn new_in(content: impl AsRef, span: TextSpan, ctx: &AllocationContext) -> Self { + let content = ctx.intern(content.as_ref()); Self { content, span } } } + +#[cfg(test)] +mod test { + use crate::{AllocationContext, TextSpan, Token, TokenStream, TokenTree}; + + #[test] + pub fn can_serde_empty_token_stream() { + let original = TokenStream::empty(); + let serialized = serde_json::to_string(&original).unwrap(); + let derived: TokenStream = serde_json::from_str(serialized.as_str()).unwrap(); + assert_eq!(original, derived); + let val: serde_json::Value = serde_json::from_str(serialized.as_str()).unwrap(); + assert_eq!( + val, + serde_json::json!({ + "tokens": [], + "metadata": { + "original_file_path": null, + "file_id": null, + "edition": null + } + }) + ); + } + + #[test] + pub fn can_serde_token_stream() { + let ctx = AllocationContext::default(); + let original = TokenStream::new(vec![ + TokenTree::Ident(Token::new_in("first", TextSpan::new(0, 1), &ctx)), + TokenTree::Ident(Token::new_in("second", TextSpan::new(2, 3), &ctx)), + TokenTree::Ident(Token::new_in("third", TextSpan::new(4, 5), &ctx)), + TokenTree::Ident(Token::new_in("fourth", TextSpan::new(6, 7), &ctx)), + ]); + let serialized = serde_json::to_string(&original).unwrap(); + let derived: TokenStream = serde_json::from_str(serialized.as_str()).unwrap(); + assert_eq!(original, derived); + let val: serde_json::Value = serde_json::from_str(serialized.as_str()).unwrap(); + assert_eq!( + val, + serde_json::json!({ + "tokens": [ + {"Ident": {"content": "first", "span": {"start": 0, "end": 1}}}, + {"Ident": {"content": "second", "span": {"start": 2, "end": 3}}}, + {"Ident": {"content": "third", "span": {"start": 4, "end": 5}}}, + {"Ident": {"content": "fourth", "span": {"start": 6, "end": 7}}}, + ], + "metadata": { + "original_file_path": null, + "file_id": null, + "edition": null + } + }) + ); + } +} diff --git a/scarb/src/compiler/plugin/proc_macro/host.rs b/scarb/src/compiler/plugin/proc_macro/host.rs index 087f1c55c..97711a9a4 100644 --- a/scarb/src/compiler/plugin/proc_macro/host.rs +++ b/scarb/src/compiler/plugin/proc_macro/host.rs @@ -14,8 +14,8 @@ use cairo_lang_diagnostics::ToOption; use cairo_lang_filesystem::db::Edition; use cairo_lang_filesystem::ids::CodeMapping; use cairo_lang_macro::{ - AuxData, Diagnostic, FullPathMarker, ProcMacroResult, Severity, TokenStream, - TokenStreamMetadata, + AllocationContext, AuxData, Diagnostic, FullPathMarker, ProcMacroResult, Severity, TokenStream, + TokenStreamMetadata, TokenTree, }; use cairo_lang_semantic::db::SemanticGroup; use cairo_lang_semantic::items::attribute::SemanticQueryAttrs; @@ -164,6 +164,7 @@ impl ProcMacroHostPlugin { let mut context = InnerAttrExpansionContext::new(self); let mut item_builder = PatchBuilder::new(db, &item_ast); let mut all_none = true; + let ctx = AllocationContext::default(); match item_ast.clone() { ast::ModuleItem::Trait(trait_ast) => { @@ -191,10 +192,11 @@ impl ProcMacroHostPlugin { let mut token_stream_builder = TokenStreamBuilder::new(db); let attrs = func.attributes(db).elements(db); - let found = self.parse_attrs(db, &mut token_stream_builder, attrs); + let found = + self.parse_attrs(db, &mut token_stream_builder, attrs, &ctx); token_stream_builder.add_node(func.declaration(db).as_syntax_node()); token_stream_builder.add_node(func.body(db).as_syntax_node()); - let token_stream = token_stream_builder.build(); + let token_stream = token_stream_builder.build(&ctx); all_none = all_none && self.do_expand_inner_attr( @@ -245,11 +247,12 @@ impl ProcMacroHostPlugin { let mut token_stream_builder = TokenStreamBuilder::new(db); let attrs = func.attributes(db).elements(db); - let found = self.parse_attrs(db, &mut token_stream_builder, attrs); + let found = + self.parse_attrs(db, &mut token_stream_builder, attrs, &ctx); token_stream_builder.add_node(func.visibility(db).as_syntax_node()); token_stream_builder.add_node(func.declaration(db).as_syntax_node()); token_stream_builder.add_node(func.body(db).as_syntax_node()); - let token_stream = token_stream_builder.build(); + let token_stream = token_stream_builder.build(&ctx); all_none = all_none && self.do_expand_inner_attr( db, @@ -311,7 +314,7 @@ impl ProcMacroHostPlugin { let result = self.instance(input.package_id).generate_code( input.expansion.name.clone(), - args.clone(), + args, token_stream.clone(), ); @@ -331,12 +334,13 @@ impl ProcMacroHostPlugin { &self, db: &dyn SyntaxGroup, item_ast: ast::ModuleItem, + ctx: &AllocationContext, ) -> (AttrExpansionFound, TokenStream) { let mut token_stream_builder = TokenStreamBuilder::new(db); let input = match item_ast.clone() { ast::ModuleItem::Trait(trait_ast) => { let attrs = trait_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs, ctx); token_stream_builder.add_node(trait_ast.visibility(db).as_syntax_node()); token_stream_builder.add_node(trait_ast.trait_kw(db).as_syntax_node()); token_stream_builder.add_node(trait_ast.name(db).as_syntax_node()); @@ -346,7 +350,7 @@ impl ProcMacroHostPlugin { } ast::ModuleItem::Impl(impl_ast) => { let attrs = impl_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs, ctx); token_stream_builder.add_node(impl_ast.visibility(db).as_syntax_node()); token_stream_builder.add_node(impl_ast.impl_kw(db).as_syntax_node()); token_stream_builder.add_node(impl_ast.name(db).as_syntax_node()); @@ -358,7 +362,7 @@ impl ProcMacroHostPlugin { } ast::ModuleItem::Module(module_ast) => { let attrs = module_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs, ctx); token_stream_builder.add_node(module_ast.visibility(db).as_syntax_node()); token_stream_builder.add_node(module_ast.module_kw(db).as_syntax_node()); token_stream_builder.add_node(module_ast.name(db).as_syntax_node()); @@ -367,7 +371,7 @@ impl ProcMacroHostPlugin { } ast::ModuleItem::FreeFunction(free_func_ast) => { let attrs = free_func_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs, ctx); token_stream_builder.add_node(free_func_ast.visibility(db).as_syntax_node()); token_stream_builder.add_node(free_func_ast.declaration(db).as_syntax_node()); token_stream_builder.add_node(free_func_ast.body(db).as_syntax_node()); @@ -375,7 +379,7 @@ impl ProcMacroHostPlugin { } ast::ModuleItem::ExternFunction(extern_func_ast) => { let attrs = extern_func_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs, ctx); token_stream_builder.add_node(extern_func_ast.visibility(db).as_syntax_node()); token_stream_builder.add_node(extern_func_ast.extern_kw(db).as_syntax_node()); token_stream_builder.add_node(extern_func_ast.declaration(db).as_syntax_node()); @@ -384,7 +388,7 @@ impl ProcMacroHostPlugin { } ast::ModuleItem::ExternType(extern_type_ast) => { let attrs = extern_type_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs, ctx); token_stream_builder.add_node(extern_type_ast.visibility(db).as_syntax_node()); token_stream_builder.add_node(extern_type_ast.extern_kw(db).as_syntax_node()); token_stream_builder.add_node(extern_type_ast.type_kw(db).as_syntax_node()); @@ -395,7 +399,7 @@ impl ProcMacroHostPlugin { } ast::ModuleItem::Struct(struct_ast) => { let attrs = struct_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs, ctx); token_stream_builder.add_node(struct_ast.visibility(db).as_syntax_node()); token_stream_builder.add_node(struct_ast.struct_kw(db).as_syntax_node()); token_stream_builder.add_node(struct_ast.name(db).as_syntax_node()); @@ -407,7 +411,7 @@ impl ProcMacroHostPlugin { } ast::ModuleItem::Enum(enum_ast) => { let attrs = enum_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs, ctx); token_stream_builder.add_node(enum_ast.visibility(db).as_syntax_node()); token_stream_builder.add_node(enum_ast.enum_kw(db).as_syntax_node()); token_stream_builder.add_node(enum_ast.name(db).as_syntax_node()); @@ -419,7 +423,7 @@ impl ProcMacroHostPlugin { } _ => AttrExpansionFound::None, }; - let token_stream = token_stream_builder.build(); + let token_stream = token_stream_builder.build(ctx); (input, token_stream) } @@ -428,6 +432,7 @@ impl ProcMacroHostPlugin { db: &dyn SyntaxGroup, builder: &mut TokenStreamBuilder<'_>, attrs: Vec, + ctx: &AllocationContext, ) -> AttrExpansionFound { // This function parses attributes of the item, // checking if those attributes correspond to a procedural macro that should be fired. @@ -452,7 +457,7 @@ impl ProcMacroHostPlugin { if expansion.is_none() { let mut args_builder = TokenStreamBuilder::new(db); args_builder.add_node(attr.arguments(db).as_syntax_node()); - let args = args_builder.build(); + let args = args_builder.build(ctx); expansion = Some((found, args, attr.stable_ptr().untyped())); // Do not add the attribute for found expansion. continue; @@ -526,7 +531,6 @@ impl ProcMacroHostPlugin { let mut token_stream_builder = TokenStreamBuilder::new(db); token_stream_builder.add_node(item_ast.as_syntax_node()); token_stream_builder.with_metadata(stream_metadata.clone()); - let token_stream = token_stream_builder.build(); let mut aux_data = EmittedAuxData::default(); let mut all_diagnostics: Vec = Vec::new(); @@ -534,12 +538,14 @@ impl ProcMacroHostPlugin { let derives = self.parse_derive(db, item_ast.clone()); let any_derives = !derives.is_empty(); + let ctx = AllocationContext::default(); let mut derived_code = PatchBuilder::new(db, &item_ast); for derive in derives { + let token_stream = token_stream_builder.build(&ctx); let result = self.instance(derive.package_id).generate_code( derive.expansion.name.clone(), TokenStream::empty(), - token_stream.clone(), + token_stream, ); // Register diagnostics. @@ -559,7 +565,13 @@ impl ProcMacroHostPlugin { continue; } - derived_code.add_str(result.token_stream.to_string().as_str()); + for token in result.token_stream.tokens { + match token { + TokenTree::Ident(token) => { + derived_code.add_str(token.content.as_ref()); + } + } + } } if any_derives { @@ -597,10 +609,11 @@ impl ProcMacroHostPlugin { token_stream: TokenStream, stable_ptr: SyntaxStablePtrId, ) -> PluginResult { + let original = token_stream.to_string(); let result = self.instance(input.package_id).generate_code( input.expansion.name.clone(), - args.clone(), - token_stream.clone(), + args, + token_stream, ); // Handle token stream. @@ -626,10 +639,7 @@ impl ProcMacroHostPlugin { // In essence, `code: None, remove_original_item: false` means `ProcMacroHost` will not be // called again for this AST item. // This optimization limits the number of generated nodes a bit. - if last - && result.aux_data.is_none() - && token_stream.to_string() == result.token_stream.to_string() - { + if last && result.aux_data.is_none() && original == result.token_stream.to_string() { return PluginResult { code: None, remove_original_item: false, @@ -852,8 +862,8 @@ impl<'a> InnerAttrExpansionContext<'a> { result: ProcMacroResult, stable_ptr: SyntaxStablePtrId, ) -> String { - let expanded = result.token_stream.to_string(); - let changed = expanded.as_str() != original; + let result_str = result.token_stream.to_string(); + let changed = result_str != original; if changed { self.host @@ -870,8 +880,9 @@ impl<'a> InnerAttrExpansionContext<'a> { self.any_changed = self.any_changed || changed; - expanded + result_str } + pub fn into_result(self, expanded: String, code_mappings: Vec) -> PluginResult { PluginResult { code: Some(PluginGeneratedFile { @@ -913,7 +924,8 @@ impl MacroPlugin for ProcMacroHostPlugin { // Expand first attribute. // Note that we only expand the first attribute, as we assume that the rest of the attributes // will be handled by a subsequent call to this function. - let (input, body) = self.parse_attribute(db, item_ast.clone()); + let ctx = AllocationContext::default(); + let (input, body) = self.parse_attribute(db, item_ast.clone(), &ctx); if let Some(result) = match input { AttrExpansionFound::Last { @@ -1024,10 +1036,11 @@ impl InlineMacroExprPlugin for ProcMacroInlinePlugin { syntax: &ast::ExprInlineMacro, _metadata: &MacroPluginMetadata<'_>, ) -> InlinePluginResult { + let ctx = AllocationContext::default(); let stable_ptr = syntax.clone().stable_ptr().untyped(); let mut token_stream_builder = TokenStreamBuilder::new(db); token_stream_builder.add_node(syntax.as_syntax_node()); - let token_stream = token_stream_builder.build(); + let token_stream = token_stream_builder.build(&ctx); let result = self.instance().generate_code( self.expansion.name.clone(), TokenStream::empty(), diff --git a/scarb/src/compiler/plugin/proc_macro/types.rs b/scarb/src/compiler/plugin/proc_macro/types.rs index dc8f5439b..30ad3e2db 100644 --- a/scarb/src/compiler/plugin/proc_macro/types.rs +++ b/scarb/src/compiler/plugin/proc_macro/types.rs @@ -1,4 +1,6 @@ -use cairo_lang_macro::{TextSpan, Token, TokenStream, TokenStreamMetadata, TokenTree}; +use cairo_lang_macro::{ + AllocationContext, TextSpan, Token, TokenStream, TokenStreamMetadata, TokenTree, +}; use cairo_lang_syntax::node::{db::SyntaxGroup, SyntaxNode}; /// Helps creating TokenStream based on multiple SyntaxNodes, @@ -26,29 +28,28 @@ impl<'a> TokenStreamBuilder<'a> { self.metadata = Some(metadata); } - pub fn build(self) -> TokenStream { - let mut result: Vec = Vec::default(); - for node in self.nodes.iter() { - let leaves = node.tokens(self.db); - let tokens = - leaves.map(|node| TokenTree::Ident(self.token_from_syntax_node(node.clone()))); - result.extend(tokens); - } + pub fn build(&self, ctx: &AllocationContext) -> TokenStream { + let result: Vec = self + .nodes + .iter() + .flat_map(|node| { + let leaves = node.tokens(self.db); + leaves.map(|node| TokenTree::Ident(self.token_from_syntax_node(node.clone(), ctx))) + }) + .collect(); - match self.metadata { - Some(metadata) => TokenStream::new(result.clone()).with_metadata(metadata.clone()), - None => TokenStream::new(result.clone()), + match self.metadata.as_ref() { + Some(metadata) => TokenStream::new(result).with_metadata(metadata.clone()), + None => TokenStream::new(result), } } - pub fn token_from_syntax_node(&self, node: SyntaxNode) -> Token { + pub fn token_from_syntax_node(&self, node: SyntaxNode, ctx: &AllocationContext) -> Token { let span = node.span(self.db).to_str_range(); - Token::new( - node.get_text(self.db), - TextSpan { - start: span.start, - end: span.end, - }, - ) + let span = TextSpan { + start: span.start, + end: span.end, + }; + Token::new_in(node.get_text(self.db), span, ctx) } } diff --git a/scarb/tests/build_cairo_plugin.rs b/scarb/tests/build_cairo_plugin.rs index 7a4af9abc..99c498134 100644 --- a/scarb/tests/build_cairo_plugin.rs +++ b/scarb/tests/build_cairo_plugin.rs @@ -734,16 +734,7 @@ fn cannot_duplicate_macros_across_packages() { fn cannot_use_undefined_macro() { let temp = TempDir::new().unwrap(); let t = temp.child("some"); - CairoPluginProjectBuilder::default() - .lib_rs(indoc! {r##" - use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro}; - - #[attribute_macro] - pub fn hello(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { - ProcMacroResult::new(token_stream) - } - "##}) - .build(&t); + CairoPluginProjectBuilder::default().build(&t); let project = temp.child("hello"); ProjectBuilder::start() .name("hello") diff --git a/scarb/tests/package.rs b/scarb/tests/package.rs index 00d3cb279..2b7233428 100644 --- a/scarb/tests/package.rs +++ b/scarb/tests/package.rs @@ -486,6 +486,7 @@ fn workspace() { #[test] fn cairo_plugin() { let t = TempDir::new().unwrap(); + // Note this will be packaged with `cairo-lang-macro` from crates, not the local one. CairoPluginProjectBuilder::default().build(&t); Scarb::quick_snapbox() diff --git a/scarb/tests/proc_macro_server.rs b/scarb/tests/proc_macro_server.rs index c2a29d2fc..8950a5b12 100644 --- a/scarb/tests/proc_macro_server.rs +++ b/scarb/tests/proc_macro_server.rs @@ -90,7 +90,7 @@ fn expand_attribute() { attr: "rename_to_very_new_name".to_string(), args: TokenStream::empty(), item: TokenStream::new(vec![TokenTree::Ident(Token::new( - "fn some_test_fn(){}".to_string(), + "fn some_test_fn(){}", TextSpan::default(), ))]), }) @@ -124,7 +124,7 @@ fn expand_derive() { let mut proc_macro_server = ProcMacroClient::new(&project); let item = TokenStream::new(vec![TokenTree::Ident(Token::new( - "fn some_test_fn(){}".to_string(), + "fn some_test_fn(){}", TextSpan::default(), ))]); @@ -181,7 +181,7 @@ fn expand_inline() { .request_and_wait::(ExpandInlineMacroParams { name: "replace_all_15_with_25".to_string(), args: TokenStream::new(vec![TokenTree::Ident(Token::new( - "struct A { field: 15 , other_field: macro_call!(12)}".to_string(), + "struct A { field: 15 , other_field: macro_call!(12)}", TextSpan::default(), ))]), })