Add support to resolve bindings to built-in functions, types and vari…

…ables (#1101)

This PR adds the infrastructure to the Bindings implementation to be
able to:
- define built-in functions, types and global variables, optionally
specifying version ranges for those
- generate an internal built-ins file that can be consumed by the
implementation when creating a `Bindings` object for a specific language
version
- resolve references to built-ins in Solidity by adding some rules to
the graph builder file to connect the built-ins parsed file and make all
definitions there available to all other ingested source units

So far, this contains just a few built-ins to test the infrastructure
and verify that all functions, types and variables can be resolved.

crates/codegen/language/definition/src/compiler/analysis/references.rs

@@ -11,8 +11,9 @@ use crate::model::SpannedItemDiscriminants::{
     self, Enum, Fragment, Keyword, Precedence, Repeated, Separated, Struct, Token, Trivia,
 };
 use crate::model::{
-    Identifier, SpannedEnumItem, SpannedEnumVariant, SpannedField, SpannedFragmentItem,
-    SpannedItem, SpannedKeywordDefinition, SpannedKeywordItem, SpannedPrecedenceExpression,
+    Identifier, SpannedBuiltIn, SpannedBuiltInField, SpannedBuiltInFunction, SpannedBuiltInType,
+    SpannedEnumItem, SpannedEnumVariant, SpannedField, SpannedFragmentItem, SpannedItem,
+    SpannedKeywordDefinition, SpannedKeywordItem, SpannedPrecedenceExpression,
     SpannedPrecedenceItem, SpannedPrecedenceOperator, SpannedPrimaryExpression,
     SpannedRepeatedItem, SpannedScanner, SpannedSeparatedItem, SpannedStructItem,
     SpannedTokenDefinition, SpannedTokenItem, SpannedTriviaItem, SpannedTriviaParser,
@@ -39,6 +40,10 @@ pub(crate) fn analyze_references(analysis: &mut Analysis) {
     for item in language.items() {
         check_item(analysis, item, &enablement);
     }
+
+    for built_in in &language.built_ins {
+        check_built_in(analysis, built_in, &enablement);
+    }
 }
 
 fn check_item(analysis: &mut Analysis, item: &SpannedItem, enablement: &VersionSet) {
@@ -396,6 +401,70 @@ fn check_reference(
     }
 }
 
+fn check_built_in(analysis: &mut Analysis, built_in: &SpannedBuiltIn, enablement: &VersionSet) {
+    match built_in {
+        SpannedBuiltIn::BuiltInFunction { item } => {
+            check_built_in_function(analysis, item, enablement);
+        }
+        SpannedBuiltIn::BuiltInType { item } => {
+            check_built_in_type(analysis, item, enablement);
+        }
+        SpannedBuiltIn::BuiltInVariable { item } => {
+            check_built_in_field(analysis, item, enablement);
+        }
+    }
+}
+
+fn check_built_in_function(
+    analysis: &mut Analysis,
+    built_in: &SpannedBuiltInFunction,
+    enablement: &VersionSet,
+) {
+    let SpannedBuiltInFunction {
+        name: _,
+        return_type: _,
+        parameters: _,
+        enabled,
+    } = built_in;
+
+    let _ = update_enablement(analysis, enablement, enabled);
+}
+
+fn check_built_in_type(
+    analysis: &mut Analysis,
+    built_in: &SpannedBuiltInType,
+    enablement: &VersionSet,
+) {
+    let SpannedBuiltInType {
+        name: _,
+        fields,
+        functions,
+        enabled,
+    } = built_in;
+
+    let enablement = update_enablement(analysis, enablement, enabled);
+
+    for field in fields {
+        check_built_in_field(analysis, field, &enablement);
+    }
+    for function in functions {
+        check_built_in_function(analysis, function, &enablement);
+    }
+}
+
+fn check_built_in_field(
+    analysis: &mut Analysis,
+    built_in: &SpannedBuiltInField,
+    enablement: &VersionSet,
+) {
+    let SpannedBuiltInField {
+        definition: _,
+        enabled,
+    } = built_in;
+
+    let _ = update_enablement(analysis, enablement, enabled);
+}
+
 fn update_enablement(
     analysis: &mut Analysis,
     existing_enablement: &VersionSet,

crates/codegen/language/definition/src/model/built_ins.rs

@@ -0,0 +1,40 @@
+use std::rc::Rc;
+
+use codegen_language_internal_macros::{derive_spanned_type, ParseInputTokens, WriteOutputTokens};
+use serde::{Deserialize, Serialize};
+use strum_macros::EnumDiscriminants;
+
+use crate::model::VersionSpecifier;
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[derive_spanned_type(Clone, Debug, EnumDiscriminants, ParseInputTokens, WriteOutputTokens)]
+pub enum BuiltIn {
+    BuiltInFunction { item: Rc<BuiltInFunction> },
+    BuiltInType { item: Rc<BuiltInType> },
+    BuiltInVariable { item: Rc<BuiltInField> },
+}
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
+pub struct BuiltInFunction {
+    pub name: String,
+    pub parameters: Vec<String>,
+    pub return_type: Option<String>,
+    pub enabled: Option<VersionSpecifier>,
+}
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
+pub struct BuiltInType {
+    pub name: String,
+    pub fields: Vec<BuiltInField>,
+    pub functions: Vec<BuiltInFunction>,
+    pub enabled: Option<VersionSpecifier>,
+}
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
+pub struct BuiltInField {
+    pub definition: String,
+    pub enabled: Option<VersionSpecifier>,
+}

crates/codegen/language/definition/src/model/manifest.rs

@@ -6,7 +6,7 @@ use indexmap::IndexSet;
 use semver::Version;
 use serde::{Deserialize, Serialize};
 
-use crate::model::{Field, Identifier, Item, TriviaParser, VersionSpecifier};
+use crate::model::{BuiltIn, Field, Identifier, Item, TriviaParser, VersionSpecifier};
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
@@ -15,6 +15,7 @@ pub struct Language {
 
     pub documentation_dir: PathBuf,
     pub binding_rules_file: PathBuf,
+    pub file_extension: Option<String>,
 
     pub root_item: Identifier,
 
@@ -24,6 +25,7 @@ pub struct Language {
     pub versions: IndexSet<Version>,
 
     pub sections: Vec<Section>,
+    pub built_ins: Vec<BuiltIn>,
 }
 
 impl Language {
@@ -104,6 +106,47 @@ impl Language {
 
         res
     }
+
+    /// Collects all versions that change the language built-ins.
+    ///
+    /// Includes the first supported version. Returns an empty set if there are
+    /// no built-ins defined.
+    pub fn collect_built_ins_versions(&self) -> BTreeSet<Version> {
+        if self.built_ins.is_empty() {
+            return BTreeSet::new();
+        }
+
+        let first = self.versions.first().unwrap().clone();
+        let mut res = BTreeSet::from_iter([first]);
+
+        let mut add_spec = |spec: &Option<VersionSpecifier>| {
+            if let Some(spec) = spec {
+                res.extend(spec.versions().cloned());
+            }
+        };
+
+        for item in &self.built_ins {
+            match item {
+                BuiltIn::BuiltInFunction { item } => {
+                    add_spec(&item.enabled);
+                }
+                BuiltIn::BuiltInType { item } => {
+                    add_spec(&item.enabled);
+                    for field in &item.fields {
+                        add_spec(&field.enabled);
+                    }
+                    for function in &item.functions {
+                        add_spec(&function.enabled);
+                    }
+                }
+                BuiltIn::BuiltInVariable { item } => {
+                    add_spec(&item.enabled);
+                }
+            }
+        }
+
+        res
+    }
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]

crates/codegen/language/definition/src/model/mod.rs

@@ -1,9 +1,11 @@
+mod built_ins;
 mod item;
 mod manifest;
 mod nonterminals;
 mod terminals;
 mod utils;
 
+pub use built_ins::*;
 pub use item::*;
 pub use manifest::*;
 pub use nonterminals::*;

crates/codegen/language/internal_macros/src/derive/spanned.rs

@@ -110,7 +110,12 @@ fn get_spanned_type(input: Type) -> Type {
     match type_name.as_str() {
         // These are model Types that have a derived 'SpannedXXX' type.
         // Let's use that instead:
-        "EnumItem"
+        "BuiltIn"
+        | "BuiltInField"
+        | "BuiltInFunction"
+        | "BuiltInParameter"
+        | "BuiltInType"
+        | "EnumItem"
         | "EnumVariant"
         | "Field"
         | "FieldDelimiters"

crates/codegen/language/tests/src/fail/definitions/duplicate_expression_name/test.rs

@@ -46,7 +46,8 @@ codegen_language_macros::compile!(Language(
                 )
             ]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/definitions/duplicate_item_name/test.rs

@@ -18,7 +18,8 @@ codegen_language_macros::compile!(Language(
                 Struct(name = Bar2, fields = (field = Required(Bar1)))
             ]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/definitions/duplicate_variant_name/test.rs

@@ -31,7 +31,8 @@ codegen_language_macros::compile!(Language(
                 )
             ]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/definitions/operator_mismatch/test.rs

@@ -36,7 +36,8 @@ codegen_language_macros::compile!(Language(
                 )
             ]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/parsing/duplicate_map_key/test.rs

@@ -27,7 +27,8 @@ codegen_language_macros::compile!(Language(
                 )
             ]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/parsing/duplicate_set_entry/test.rs

@@ -17,7 +17,8 @@ codegen_language_macros::compile!(Language(
                 definitions = [TokenDefinition(scanner = Atom("bar"))]
             )]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/parsing/empty_string/test.rs

@@ -17,7 +17,8 @@ codegen_language_macros::compile!(Language(
                 definitions = [TokenDefinition(scanner = Atom(""))]
             )]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/parsing/missing_field/test.rs

@@ -11,7 +11,8 @@ codegen_language_macros::compile!(Language(
     sections = [Section(
         // title = "Section One"
         topics = []
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/parsing/unrecognized_field/test.rs

@@ -18,7 +18,8 @@ codegen_language_macros::compile!(Language(
             )],
             unrecognized_field = true
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/parsing/unrecognized_variant/test.rs

@@ -20,7 +20,8 @@ codegen_language_macros::compile!(Language(
             ),
             Topic(title = "Topic Two", items = [Unrecognized(true)])
         ]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/reachability/unreachable/test.rs

@@ -21,7 +21,8 @@ codegen_language_macros::compile!(Language(
                 Struct(name = Baz2, fields = (field = Required(Baz1)))
             ]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/reachability/unused_version/test.rs

@@ -23,7 +23,8 @@ codegen_language_macros::compile!(Language(
                 )
             ]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/references/disabled_too_late/test.rs

@@ -34,7 +34,8 @@ codegen_language_macros::compile!(Language(
                 )
             ]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/references/enabled_too_early/test.rs

@@ -34,7 +34,8 @@ codegen_language_macros::compile!(Language(
                 )
             ]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/references/invalid_filter/test.rs

@@ -17,7 +17,8 @@ codegen_language_macros::compile!(Language(
                 Fragment(name = Baz, scanner = Atom("baz"))
             ]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}

crates/codegen/language/tests/src/fail/references/invalid_version/test.rs

@@ -46,7 +46,8 @@ codegen_language_macros::compile!(Language(
                 )
             ]
         )]
-    )]
+    )],
+    built_ins = []
 ));
 
 fn main() {}