feat: Sync commits from aztec-packages (#4068)

Development from
[aztec-packages](https://github.com/AztecProtocol/aztec-packages).

---------

Co-authored-by: ludamad <adam.domurad@gmail.com>
Co-authored-by: ludamad <adam@aztecprotocol.com>
Co-authored-by: kevaundray <kevtheappdev@gmail.com>
Co-authored-by: sirasistant <sirasistant@gmail.com>
Co-authored-by: Gregorio Juliana <gregojquiros@gmail.com>
Co-authored-by: Tom French <tom@tomfren.ch>
Co-authored-by: Maxim Vezenov <mvezenov@gmail.com>
Co-authored-by: Jan Beneš <janbenes1234@gmail.com>
Co-authored-by: Charlie Lye <karl.lye@gmail.com>
Co-authored-by: Tom French <15848336+TomAFrench@users.noreply.github.com>
Co-authored-by: ledwards2225 <l.edwards.d@gmail.com>
Co-authored-by: ledwards2225 <98505400+ledwards2225@users.noreply.github.com>
Co-authored-by: Santiago Palladino <santiago@aztecprotocol.com>
Co-authored-by: James Zaki <james.zaki@proton.me>

Cargo.toml

@@ -123,6 +123,7 @@ num-traits = "0.2"
 similar-asserts = "1.5.0"
 tempfile = "3.6.0"
 jsonrpc = { version = "0.16.0", features = ["minreq_http"] }
+flate2 = "1.0.24"
 
 tracing = "0.1.40"
 tracing-web = "0.1.3"

README.md

@@ -58,7 +58,7 @@ This crate's minimum supported rustc version is 1.71.1.
 
 ## Working on this project
 
-This project uses [Nix](https://nixos.org/) and [direnv](https://direnv.net/) to streamline the development experience. Please follow [our guidelines](https://noir-lang.org/docs/dev/getting_started/installation/other_install_methods#option-3-compile-from-source) to setup your environment for working on the project.
+This project uses [Nix](https://nixos.org/) and [direnv](https://direnv.net/) to streamline the development experience. Please follow [our guidelines](https://noir-lang.org/docs/getting_started/installation/other_install_methods#option-3-compile-from-source) to setup your environment for working on the project.
 
 ### Building against a different local/remote version of Barretenberg
 

acvm-repo/acir/Cargo.toml

@@ -17,7 +17,7 @@ acir_field.workspace = true
 brillig.workspace = true
 serde.workspace = true
 thiserror.workspace = true
-flate2 = "1.0.24"
+flate2.workspace = true
 bincode.workspace = true
 base64.workspace = true
 

bootstrap_cache.sh

@@ -5,6 +5,7 @@ cd "$(dirname "$0")"
 source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving noir packages from remote cache...\033[0m"
-extract_repo noir-packages /usr/src/noir/packages ./noir
+extract_repo noir-packages /usr/src/noir/packages ./
 echo -e "\033[1mRetrieving nargo from remote cache...\033[0m"
-extract_repo noir /usr/src/noir/target/release ./noir/target
+extract_repo noir /usr/src/noir/target/release ./target/
+

compiler/noirc_errors/Cargo.toml

@@ -15,4 +15,7 @@ fm.workspace = true
 chumsky.workspace = true
 serde.workspace = true
 serde_with = "3.2.0"
-tracing.workspace = true
+tracing.workspace = true
+flate2.workspace = true
+serde_json.workspace = true
+base64.workspace = true

compiler/noirc_errors/src/debug_info.rs

@@ -1,14 +1,24 @@
 use acvm::acir::circuit::OpcodeLocation;
 use acvm::compiler::AcirTransformationMap;
 
+use base64::Engine;
+use flate2::read::DeflateDecoder;
+use flate2::write::DeflateEncoder;
+use flate2::Compression;
+use serde::Deserializer;
+use serde::Serializer;
 use serde_with::serde_as;
 use serde_with::DisplayFromStr;
 use std::collections::BTreeMap;
 use std::collections::HashMap;
+use std::io::Read;
+use std::io::Write;
 use std::mem;
 
 use crate::Location;
-use serde::{Deserialize, Serialize};
+use serde::{
+    de::Error as DeserializationError, ser::Error as SerializationError, Deserialize, Serialize,
+};
 
 #[serde_as]
 #[derive(Default, Debug, Clone, Deserialize, Serialize)]
@@ -86,4 +96,40 @@ impl DebugInfo {
 
         counted_opcodes
     }
+
+    pub fn serialize_compressed_base64_json<S>(
+        debug_info: &DebugInfo,
+        s: S,
+    ) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let json_str = serde_json::to_string(debug_info).map_err(S::Error::custom)?;
+
+        let mut encoder = DeflateEncoder::new(Vec::new(), Compression::default());
+        encoder.write_all(json_str.as_bytes()).map_err(S::Error::custom)?;
+        let compressed_data = encoder.finish().map_err(S::Error::custom)?;
+
+        let encoded_b64 = base64::prelude::BASE64_STANDARD.encode(compressed_data);
+        s.serialize_str(&encoded_b64)
+    }
+
+    pub fn deserialize_compressed_base64_json<'de, D>(
+        deserializer: D,
+    ) -> Result<DebugInfo, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let encoded_b64: String = Deserialize::deserialize(deserializer)?;
+
+        let compressed_data =
+            base64::prelude::BASE64_STANDARD.decode(encoded_b64).map_err(D::Error::custom)?;
+
+        let mut decoder = DeflateDecoder::new(&compressed_data[..]);
+        let mut decompressed_data = Vec::new();
+        decoder.read_to_end(&mut decompressed_data).map_err(D::Error::custom)?;
+
+        let json_str = String::from_utf8(decompressed_data).map_err(D::Error::custom)?;
+        serde_json::from_str(&json_str).map_err(D::Error::custom)
+    }
 }

compiler/wasm/Cargo.toml

@@ -18,6 +18,7 @@ nargo.workspace = true
 noirc_driver.workspace = true
 noirc_frontend.workspace = true
 noirc_errors.workspace = true
+noirc_evaluator.workspace = true
 wasm-bindgen.workspace = true
 serde.workspace = true
 js-sys.workspace = true

compiler/wasm/package.json

@@ -34,6 +34,7 @@
     "test": "yarn test:build_fixtures && yarn test:node && yarn test:browser",
     "test:build_fixtures": "./scripts/build-fixtures.sh",
     "test:browser": "web-test-runner",
+    "test:browser:docker": "docker run --rm -v $(cd ../.. && pwd):/usr/src/noir -w /usr/src/noir/compiler/wasm mcr.microsoft.com/playwright:v1.40.0-jammy yarn test:browser",
     "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
     "clean": "rm -rf ./build ./target ./dist public/fixtures/simple/target public/fixtures/with-deps/target",
     "nightly:version": "jq --arg new_version \"-$(git rev-parse --short HEAD)$1\" '.version = .version + $new_version' package.json > package-tmp.json && mv package-tmp.json package.json",
@@ -49,6 +50,7 @@
     "@types/mocha": "^10.0.6",
     "@types/mocha-each": "^2",
     "@types/node": "^20.10.5",
+    "@types/pako": "^2",
     "@types/path-browserify": "^1",
     "@types/readable-stream": "^4",
     "@types/sinon": "^17",
@@ -76,5 +78,8 @@
     "url": "^0.11.3",
     "webpack": "^5.49.0",
     "webpack-cli": "^4.7.2"
+  },
+  "dependencies": {
+    "pako": "^2.1.0"
   }
 }

compiler/wasm/src/compile.rs

@@ -3,14 +3,14 @@ use gloo_utils::format::JsValueSerdeExt;
 use js_sys::{JsString, Object};
 use nargo::artifacts::{
     contract::{ContractArtifact, ContractFunctionArtifact},
-    debug::DebugArtifact,
     program::ProgramArtifact,
 };
 use noirc_driver::{
     add_dep, compile_contract, compile_main, file_manager_with_stdlib, prepare_crate,
     prepare_dependency, CompileOptions, CompiledContract, CompiledProgram,
     NOIR_ARTIFACT_VERSION_STRING,
 };
+use noirc_evaluator::errors::SsaReport;
 use noirc_frontend::{
     graph::{CrateId, CrateName},
     hir::{def_map::parse_file, Context, ParsedFiles},
@@ -28,35 +28,30 @@ export type DependencyGraph = {
     library_dependencies: Readonly<Record<string, readonly string[]>>;
 }
 
-export type CompiledContract = {
+export type ContractArtifact = {
     noir_version: string;
     name: string;
     functions: Array<any>;
     events: Array<any>;
+    file_map: Record<number, any>;
 };
 
-export type CompiledProgram = {
+export type ProgramArtifact = {
     noir_version: string;
+    hash: number;
     abi: any;
     bytecode: string;
+    debug_symbols: any;
+    file_map: Record<number, any>;
 }
 
-export type DebugArtifact = {
-    debug_symbols: Array<any>;
-    file_map: Record<number, any>;
-    warnings: Array<any>;
-};
+type WarningsCompileResult = { warnings: Array<any>; };
 
-export type CompileResult = (
-    | {
-        contract: CompiledContract;
-        debug: DebugArtifact;
-    }
-    | {
-        program: CompiledProgram;
-        debug: DebugArtifact;
-    }
-);
+export type ContractCompileResult = { contract: CompiledContract; } & WarningsCompileResult;
+
+export type ProgramCompileResult = { program: CompiledProgram; } & WarningsCompileResult;
+
+export type CompileResult = ContractCompileResult | ProgramCompileResult;
 "#;
 
 #[wasm_bindgen]
@@ -76,38 +71,36 @@ extern "C" {
 impl JsCompileResult {
     const CONTRACT_PROP: &'static str = "contract";
     const PROGRAM_PROP: &'static str = "program";
-    const DEBUG_PROP: &'static str = "debug";
+    const WARNINGS_PROP: &'static str = "warnings";
 
     pub fn new(resp: CompileResult) -> JsCompileResult {
         let obj = JsCompileResult::constructor();
         match resp {
-            CompileResult::Contract { contract, debug } => {
+            CompileResult::Contract { contract, warnings } => {
                 js_sys::Reflect::set(
                     &obj,
                     &JsString::from(JsCompileResult::CONTRACT_PROP),
                     &<JsValue as JsValueSerdeExt>::from_serde(&contract).unwrap(),
                 )
                 .unwrap();
-
                 js_sys::Reflect::set(
                     &obj,
-                    &JsString::from(JsCompileResult::DEBUG_PROP),
-                    &<JsValue as JsValueSerdeExt>::from_serde(&debug).unwrap(),
+                    &JsString::from(JsCompileResult::WARNINGS_PROP),
+                    &<JsValue as JsValueSerdeExt>::from_serde(&warnings).unwrap(),
                 )
                 .unwrap();
             }
-            CompileResult::Program { program, debug } => {
+            CompileResult::Program { program, warnings } => {
                 js_sys::Reflect::set(
                     &obj,
                     &JsString::from(JsCompileResult::PROGRAM_PROP),
                     &<JsValue as JsValueSerdeExt>::from_serde(&program).unwrap(),
                 )
                 .unwrap();
-
                 js_sys::Reflect::set(
                     &obj,
-                    &JsString::from(JsCompileResult::DEBUG_PROP),
-                    &<JsValue as JsValueSerdeExt>::from_serde(&debug).unwrap(),
+                    &JsString::from(JsCompileResult::WARNINGS_PROP),
+                    &<JsValue as JsValueSerdeExt>::from_serde(&warnings).unwrap(),
                 )
                 .unwrap();
             }
@@ -152,8 +145,8 @@ pub(crate) fn parse_all(fm: &FileManager) -> ParsedFiles {
 }
 
 pub enum CompileResult {
-    Contract { contract: ContractArtifact, debug: DebugArtifact },
-    Program { program: ProgramArtifact, debug: DebugArtifact },
+    Contract { contract: ContractArtifact, warnings: Vec<SsaReport> },
+    Program { program: ProgramArtifact, warnings: Vec<SsaReport> },
 }
 
 #[wasm_bindgen]
@@ -277,31 +270,22 @@ fn add_noir_lib(context: &mut Context, library_name: &CrateName) -> CrateId {
 }
 
 pub(crate) fn generate_program_artifact(program: CompiledProgram) -> CompileResult {
-    let debug_artifact = DebugArtifact {
-        debug_symbols: vec![program.debug.clone()],
-        file_map: program.file_map.clone(),
-        warnings: program.warnings.clone(),
-    };
-
-    CompileResult::Program { program: program.into(), debug: debug_artifact }
+    let warnings = program.warnings.clone();
+    CompileResult::Program { program: program.into(), warnings }
 }
 
 pub(crate) fn generate_contract_artifact(contract: CompiledContract) -> CompileResult {
-    let debug_artifact = DebugArtifact {
-        debug_symbols: contract.functions.iter().map(|function| function.debug.clone()).collect(),
-        file_map: contract.file_map,
-        warnings: contract.warnings,
-    };
     let functions = contract.functions.into_iter().map(ContractFunctionArtifact::from).collect();
 
     let contract_artifact = ContractArtifact {
         noir_version: String::from(NOIR_ARTIFACT_VERSION_STRING),
         name: contract.name,
         functions,
         events: contract.events,
+        file_map: contract.file_map,
     };
 
-    CompileResult::Contract { contract: contract_artifact, debug: debug_artifact }
+    CompileResult::Contract { contract: contract_artifact, warnings: contract.warnings }
 }
 
 #[cfg(test)]

compiler/wasm/src/index.cts

@@ -3,6 +3,7 @@ import { createNodejsFileManager } from './noir/file-manager/nodejs-file-manager
 import { NoirWasmCompiler } from './noir/noir-wasm-compiler';
 import { LogData, LogFn } from './utils';
 import { CompilationResult } from './types/noir_artifact';
+import { inflateDebugSymbols } from './noir/debug';
 
 async function compile(
   fileManager: FileManager,
@@ -46,4 +47,4 @@ async function compile(
 
 const createFileManager = createNodejsFileManager;
 
-export { compile, createFileManager, CompilationResult };
+export { compile, createFileManager, inflateDebugSymbols, CompilationResult };

compiler/wasm/src/index.mts

@@ -3,6 +3,7 @@ import { createNodejsFileManager } from './noir/file-manager/nodejs-file-manager
 import { NoirWasmCompiler } from './noir/noir-wasm-compiler';
 import { LogData, LogFn } from './utils';
 import { CompilationResult } from './types/noir_artifact';
+import { inflateDebugSymbols } from './noir/debug';
 
 async function compile(
   fileManager: FileManager,
@@ -48,4 +49,4 @@ async function compile(
 
 const createFileManager = createNodejsFileManager;
 
-export { compile, createFileManager, CompilationResult };
+export { compile, createFileManager, inflateDebugSymbols, CompilationResult };

compiler/wasm/src/noir/debug.ts

@@ -0,0 +1,6 @@
+import { inflate } from 'pako';
+
+/** Decompresses and decodes the debug symbols */
+export function inflateDebugSymbols(debugSymbols: string) {
+  return JSON.parse(inflate(Buffer.from(debugSymbols, 'base64'), { to: 'string', raw: true }));
+}