chore: Sync to noir-lang/noir

chore: bootstrap improvements. (AztecProtocol/aztec-packages#4711)
refactor: using Tuples in `TxEffect`s and renaming note commitments (AztecProtocol/aztec-packages#4717)
chore: Build nargo against Ubuntu 20 for better compatability (AztecProtocol/aztec-packages#4710)
fix(dsl): Add full recursive verification test (AztecProtocol/aztec-packages#4658)
feat!: autogenerate compute_note_hash_and_nullifier (AztecProtocol/aztec-packages#4610)
chore: add struct for each bigint modulus (AztecProtocol/aztec-packages#4422)
chore: switch noir pull to master branch (AztecProtocol/aztec-packages#4581)

Dockerfile

@@ -1,12 +1,12 @@
-FROM rust:bookworm
+FROM rust:bullseye
 WORKDIR /usr/src/noir
 COPY . .
 RUN ./scripts/bootstrap_native.sh
 
 # When running the container, mount the users home directory to same location.
-FROM ubuntu:lunar
+FROM ubuntu:focal
 # Install Tini as nargo doesn't handle signals properly.
 # Install git as nargo needs it to clone.
 RUN apt-get update && apt-get install -y git tini && rm -rf /var/lib/apt/lists/* && apt-get clean
 COPY --from=0 /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo
-ENTRYPOINT ["/usr/bin/tini", "--", "/usr/src/noir/target/release/nargo"]
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/src/noir/target/release/nargo"]

aztec_macros/Cargo.toml

@@ -11,5 +11,6 @@ repository.workspace = true
 
 [dependencies]
 noirc_frontend.workspace = true
+noirc_errors.workspace = true
 iter-extended.workspace = true
 convert_case = "0.6.0"

aztec_macros/src/lib.rs

@@ -3,6 +3,8 @@ use std::vec;
 
 use convert_case::{Case, Casing};
 use iter_extended::vecmap;
+use noirc_frontend::hir::def_collector::dc_crate::{UnresolvedFunctions, UnresolvedTraitImpl};
+use noirc_frontend::hir::def_map::{LocalModuleId, ModuleId};
 use noirc_frontend::macros_api::FieldElement;
 use noirc_frontend::macros_api::{
     BlockExpression, CallExpression, CastExpression, Distinctness, Expression, ExpressionKind,
@@ -16,9 +18,10 @@ use noirc_frontend::macros_api::{
 use noirc_frontend::macros_api::{CrateId, FileId};
 use noirc_frontend::macros_api::{MacroError, MacroProcessor};
 use noirc_frontend::macros_api::{ModuleDefId, NodeInterner, SortedModule, StructId};
-use noirc_frontend::node_interner::{TraitId, TraitImplKind};
+use noirc_frontend::node_interner::{FuncId, TraitId, TraitImplId, TraitImplKind};
 use noirc_frontend::Lambda;
-
+use noirc_frontend::macros_api::parse_program;
+use noirc_errors::Location;
 pub struct AztecMacro;
 
 impl MacroProcessor for AztecMacro {
@@ -31,6 +34,20 @@ impl MacroProcessor for AztecMacro {
         transform(ast, crate_id, context)
     }
 
+    fn process_unresolved_traits_impls(
+        &self,
+        crate_id: &CrateId,
+        context: &mut HirContext,
+        unresolved_traits_impls: &Vec<UnresolvedTraitImpl>,
+        collected_functions: &mut Vec<UnresolvedFunctions>,
+    ) -> Result<(), (MacroError, FileId)> {
+        if has_aztec_dependency(crate_id, context) {
+            inject_compute_note_hash_and_nullifier(crate_id, context, unresolved_traits_impls, collected_functions)
+        } else {
+            Ok(())
+        }
+    }
+
     fn process_typed_ast(
         &self,
         crate_id: &CrateId,
@@ -46,7 +63,6 @@ const MAX_CONTRACT_FUNCTIONS: usize = 2_usize.pow(FUNCTION_TREE_HEIGHT);
 #[derive(Debug, Clone)]
 pub enum AztecMacroError {
     AztecDepNotFound,
-    ComputeNoteHashAndNullifierNotFound { span: Span },
     ContractHasTooManyFunctions { span: Span },
     ContractConstructorMissing { span: Span },
     UnsupportedFunctionArgumentType { span: Span, typ: UnresolvedTypeData },
@@ -63,11 +79,6 @@ impl From<AztecMacroError> for MacroError {
                 secondary_message: None,
                 span: None,
             },
-            AztecMacroError::ComputeNoteHashAndNullifierNotFound { span } => MacroError {
-                primary_message: "compute_note_hash_and_nullifier function not found. Define it in your contract. For more information go to https://docs.aztec.network/developers/debugging/aztecnr-errors#compute_note_hash_and_nullifier-function-not-found-define-it-in-your-contract".to_owned(),
-                secondary_message: None,
-                span: Some(span),
-            },
             AztecMacroError::ContractHasTooManyFunctions { span } => MacroError {
                 primary_message: format!("Contract can only have a maximum of {} functions", MAX_CONTRACT_FUNCTIONS),
                 secondary_message: None,
@@ -313,15 +324,17 @@ fn check_for_aztec_dependency(
     crate_id: &CrateId,
     context: &HirContext,
 ) -> Result<(), (MacroError, FileId)> {
-    let crate_graph = &context.crate_graph[crate_id];
-    let has_aztec_dependency = crate_graph.dependencies.iter().any(|dep| dep.as_name() == "aztec");
-    if has_aztec_dependency {
+    if has_aztec_dependency(crate_id, context) {
         Ok(())
     } else {
-        Err((AztecMacroError::AztecDepNotFound.into(), crate_graph.root_file_id))
+        Err((AztecMacroError::AztecDepNotFound.into(), context.crate_graph[crate_id].root_file_id))
     }
 }
 
+fn has_aztec_dependency(crate_id: &CrateId, context: &HirContext) -> bool {
+    context.crate_graph[crate_id].dependencies.iter().any(|dep| dep.as_name() == "aztec")
+}
+
 // Check to see if the user has defined a storage struct
 fn check_for_storage_definition(module: &SortedModule) -> bool {
     module.types.iter().any(|r#struct| r#struct.name.0.contents == "Storage")
@@ -338,27 +351,27 @@ fn check_for_storage_implementation(module: &SortedModule) -> bool {
 }
 
 // Check if "compute_note_hash_and_nullifier(AztecAddress,Field,Field,Field,[Field; N]) -> [Field; 4]" is defined
-fn check_for_compute_note_hash_and_nullifier_definition(module: &SortedModule) -> bool {
-    module.functions.iter().any(|func| {
-        func.def.name.0.contents == "compute_note_hash_and_nullifier"
-                && func.def.parameters.len() == 5
-                && match &func.def.parameters[0].typ.typ {
+fn check_for_compute_note_hash_and_nullifier_definition(functions_data: &Vec<(LocalModuleId, FuncId, NoirFunction)>, module_id: LocalModuleId) -> bool {
+    functions_data.iter().filter(|func_data| func_data.0 == module_id).any(|func_data| {
+        func_data.2.def.name.0.contents == "compute_note_hash_and_nullifier"
+                && func_data.2.def.parameters.len() == 5
+                && match &func_data.2.def.parameters[0].typ.typ {
                     UnresolvedTypeData::Named(path, _, _) => path.segments.last().unwrap().0.contents == "AztecAddress",
                     _ => false,
                 }
-                && func.def.parameters[1].typ.typ == UnresolvedTypeData::FieldElement
-                && func.def.parameters[2].typ.typ == UnresolvedTypeData::FieldElement
-                && func.def.parameters[3].typ.typ == UnresolvedTypeData::FieldElement
+                && func_data.2.def.parameters[1].typ.typ == UnresolvedTypeData::FieldElement
+                && func_data.2.def.parameters[2].typ.typ == UnresolvedTypeData::FieldElement
+                && func_data.2.def.parameters[3].typ.typ == UnresolvedTypeData::FieldElement
                 // checks if the 5th parameter is an array and the Box<UnresolvedType> in
                 // Array(Option<UnresolvedTypeExpression>, Box<UnresolvedType>) contains only fields
-                && match &func.def.parameters[4].typ.typ {
+                && match &func_data.2.def.parameters[4].typ.typ {
                     UnresolvedTypeData::Array(_, inner_type) => {
                         matches!(inner_type.typ, UnresolvedTypeData::FieldElement)
                     },
                     _ => false,
                 }
                 // We check the return type the same way as we did the 5th parameter
-                && match &func.def.return_type {
+                && match &func_data.2.def.return_type {
                     FunctionReturnType::Default(_) => false,
                     FunctionReturnType::Ty(unresolved_type) => {
                         match &unresolved_type.typ {
@@ -401,13 +414,6 @@ fn transform_module(
         generate_storage_implementation(module).map_err(|err| (err, crate_graph.root_file_id))?;
     }
 
-    if storage_defined && !check_for_compute_note_hash_and_nullifier_definition(module) {
-        return Err((
-            AztecMacroError::ComputeNoteHashAndNullifierNotFound { span: Span::default() },
-            crate_graph.root_file_id,
-        ));
-    }
-
     for structure in module.types.iter() {
         if structure.attributes.iter().any(|attr| matches!(attr, SecondaryAttribute::Event)) {
             module.impls.push(generate_selector_impl(structure));
@@ -596,7 +602,7 @@ fn generate_storage_implementation(module: &mut SortedModule) -> Result<(), Azte
 /// If it does, it will insert the following things:
 /// - A new Input that is provided for a kernel app circuit, named: {Public/Private}ContextInputs
 /// - Hashes all of the function input variables
-///     - This instantiates a helper function  
+///     - This instantiates a helper function
 fn transform_function(
     ty: &str,
     func: &mut NoirFunction,
@@ -1601,3 +1607,165 @@ fn event_signature(event: &StructType) -> String {
     let fields = vecmap(event.get_fields(&[]), |(_, typ)| signature_of_type(&typ));
     format!("{}({})", event.name.0.contents, fields.join(","))
 }
+
+fn inject_compute_note_hash_and_nullifier(
+    crate_id: &CrateId,
+    context: &mut HirContext,
+    unresolved_traits_impls: &Vec<UnresolvedTraitImpl>,
+    collected_functions: &mut Vec<UnresolvedFunctions>,
+) -> Result<(), (MacroError, FileId)> {
+    // We first fetch modules in this crate which correspond to contracts, along with their file id.
+    let contract_module_file_ids: Vec<(LocalModuleId, FileId)> = context.def_map(crate_id).expect("ICE: Missing crate in def_map")
+        .modules().iter()
+        .filter(|(_, module)| module.is_contract)
+        .map(|(idx, module)| (LocalModuleId(idx), module.location.file))
+        .collect();
+
+    // If the current crate does not contain a contract module we simply skip it.
+    if contract_module_file_ids.len() == 0 {
+        return Ok(());
+    } else if contract_module_file_ids.len() != 1 {
+        panic!("Found multiple contracts in the same crate");
+    }
+
+    let (module_id, file_id) = contract_module_file_ids[0];
+
+    // If compute_note_hash_and_nullifier is already defined by the user, we skip auto-generation in order to provide an
+    // escape hatch for this mechanism.
+    // TODO(#4647): improve this diagnosis and error messaging.
+    if collected_functions.iter().any(|coll_funcs_data| check_for_compute_note_hash_and_nullifier_definition(&coll_funcs_data.functions, module_id)) {
+        return Ok(());
+    }
+
+    // In order to implement compute_note_hash_and_nullifier, we need to know all of the different note types the
+    // contract might use. These are the types that implement the NoteInterface trait, which provides the
+    // get_note_type_id function.
+    let note_types = fetch_struct_trait_impls(context, unresolved_traits_impls, "NoteInterface");
+
+    // We can now generate a version of compute_note_hash_and_nullifier tailored for the contract in this crate.
+    let func = generate_compute_note_hash_and_nullifier(&note_types);
+
+    // And inject the newly created function into the contract.
+
+    // TODO(#4373): We don't have a reasonable location for the source code of this autogenerated function, so we simply 
+    // pass an empty span. This function should not produce errors anyway so this should not matter.
+    let location = Location::new(Span::empty(0), file_id);
+
+    // These are the same things the ModCollector does when collecting functions: we push the function to the
+    // NodeInterner, declare it in the module (which checks for duplicate definitions), and finally add it to the list
+    // on collected but unresolved functions.
+
+    let func_id = context.def_interner.push_empty_fn();
+    context.def_interner.push_function(func_id, &func.def, ModuleId { krate: *crate_id, local_id: module_id }, location);
+
+    context.def_map_mut(crate_id).unwrap()
+        .modules_mut()[module_id.0]
+        .declare_function(
+            func.name_ident().clone(), func_id
+        ).expect(
+            "Failed to declare the autogenerated compute_note_hash_and_nullifier function, likely due to a duplicate definition. See https://github.com/AztecProtocol/aztec-packages/issues/4647."
+        );
+
+    collected_functions.iter_mut()
+        .find(|fns| fns.file_id == file_id).expect("ICE: no functions found in contract file")
+        .push_fn(module_id, func_id, func.clone());
+
+    Ok(())
+}
+
+// Fetches the name of all structs that implement trait_name, both in the current crate and all of its dependencies.
+fn fetch_struct_trait_impls(
+    context: &mut HirContext,
+    unresolved_traits_impls: &Vec<UnresolvedTraitImpl>,
+    trait_name: &str
+) -> Vec<String> {
+    let mut struct_typenames: Vec<String> = Vec::new();
+
+    // These structs can be declared in either external crates or the current one. External crates that contain 
+    // dependencies have already been processed and resolved, but are available here via the NodeInterner. Note that
+    // crates on which the current crate does not depend on may not have been processed, and will be ignored.
+    for trait_impl_id in 0..(&context.def_interner.next_trait_impl_id()).0 {
+        let trait_impl = &context.def_interner.get_trait_implementation(TraitImplId(trait_impl_id));
+
+        if trait_impl.borrow().ident.0.contents == *trait_name {
+            if let Type::Struct(s, _) = &trait_impl.borrow().typ {
+                struct_typenames.push(s.borrow().name.0.contents.clone());
+            } else {
+                panic!("Found impl for {} on non-Struct", trait_name);
+            }
+        }
+    }
+
+    // This crate's traits and impls have not yet been resolved, so we look for impls in unresolved_trait_impls.
+    struct_typenames.extend(unresolved_traits_impls.iter()
+        .filter(|trait_impl| 
+            trait_impl.trait_path.segments.last().expect("ICE: empty trait_impl path").0.contents == *trait_name)
+        .filter_map(|trait_impl| match &trait_impl.object_type.typ {
+            UnresolvedTypeData::Named(path, _, _) => Some(path.segments.last().unwrap().0.contents.clone()),
+            _ => None,
+        }));
+
+    struct_typenames
+}
+
+fn generate_compute_note_hash_and_nullifier(note_types: &Vec<String>) -> NoirFunction {
+    let function_source = generate_compute_note_hash_and_nullifier_source(note_types);
+
+    let (function_ast, errors) = parse_program(&function_source);
+    if !errors.is_empty() {
+        dbg!(errors.clone());
+    }
+    assert_eq!(errors.len(), 0, "Failed to parse Noir macro code. This is either a bug in the compiler or the Noir macro code");
+
+    let mut function_ast = function_ast.into_sorted();
+    function_ast.functions.remove(0)
+}
+
+fn generate_compute_note_hash_and_nullifier_source(note_types: &Vec<String>) -> String {
+    // TODO(#4649): The serialized_note parameter is a fixed-size array, but we don't know what length it should have. 
+    // For now we hardcode it to 20, which is the same as MAX_NOTE_FIELDS_LENGTH.
+
+    if note_types.len() == 0 {
+        // TODO(#4520): Even if the contract does not include any notes, other parts of the stack expect for this 
+        // function to exist, so we include a dummy version. We likely should error out here instead.
+        "
+        unconstrained fn compute_note_hash_and_nullifier(
+            contract_address: AztecAddress,
+            nonce: Field,
+            storage_slot: Field,
+            note_type_id: Field,
+            serialized_note: [Field; 20]
+        ) -> pub [Field; 4] {
+            [0, 0, 0, 0]
+        }".to_string()
+    } else {
+        // For contracts that include notes we do a simple if-else chain comparing note_type_id with the different
+        // get_note_type_id of each of the note types.
+
+        let if_statements: Vec<String> = note_types.iter().map(|note_type| format!(
+            "if (note_type_id == {0}::get_note_type_id()) {{
+                note_utils::compute_note_hash_and_nullifier({0}::deserialize_content, note_header, serialized_note)
+            }}"
+        , note_type)).collect();
+
+        // TODO(#4520): error out on the else instead of returning a zero array
+        let full_if_statement = if_statements.join(" else ") + "
+            else {
+                [0, 0, 0, 0]
+            }";
+
+        format!("
+            unconstrained fn compute_note_hash_and_nullifier(
+                contract_address: AztecAddress,
+                nonce: Field,
+                storage_slot: Field,
+                note_type_id: Field,
+                serialized_note: [Field; 20]
+            ) -> pub [Field; 4] {{
+                let note_header = NoteHeader::new(contract_address, nonce, storage_slot);
+
+                {}
+            }}", full_if_statement)
+}
+
+}

bootstrap.sh

@@ -15,5 +15,8 @@ if [ -n "$CMD" ]; then
   fi
 fi
 
+# Attempt to just pull artefacts from CI and exit on success.
+./bootstrap_cache.sh && exit
+
 ./scripts/bootstrap_native.sh
 ./scripts/bootstrap_packages.sh

bootstrap_cache.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -eu
 
+[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1
+
 cd "$(dirname "$0")"
 source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
@@ -9,3 +11,5 @@ extract_repo noir-packages /usr/src/noir/packages ./
 echo -e "\033[1mRetrieving nargo from remote cache...\033[0m"
 extract_repo noir /usr/src/noir/target/release ./target/
 
+remove_old_images noir-packages
+remove_old_images noir

compiler/noirc_frontend/src/hir/def_collector/dc_crate.rs

@@ -256,6 +256,15 @@ impl DefCollector {
         // Add the current crate to the collection of DefMaps
         context.def_maps.insert(crate_id, def_collector.def_map);
 
+        // TODO(#4653): generalize this function
+        for macro_processor in &macro_processors {
+            macro_processor.process_unresolved_traits_impls(&crate_id, context, &def_collector.collected_traits_impls, &mut def_collector.collected_functions).unwrap_or_else(
+                |(macro_err, file_id)| {
+                    errors.push((macro_err.into(), file_id));
+                },
+            );
+        }
+
         inject_prelude(crate_id, context, crate_root, &mut def_collector.collected_imports);
         for submodule in submodules {
             inject_prelude(

compiler/noirc_frontend/src/hir/def_map/mod.rs

@@ -135,6 +135,11 @@ impl CrateDefMap {
     pub fn modules(&self) -> &Arena<ModuleData> {
         &self.modules
     }
+
+    pub fn modules_mut(&mut self) -> &mut Arena<ModuleData> {
+        &mut self.modules
+    }
+
     pub fn krate(&self) -> CrateId {
         self.krate
     }