[ty] defer inference of legacy TypeVar bound/constraints/defaults

Author: carljm

crates/ty_python_semantic/resources/mdtest/generics/legacy/variables.md

@@ -231,4 +231,69 @@ def constrained(x: T_constrained):
     reveal_type(type(x))  # revealed: type[int] | type[str]
 ```
 
+## Cycles
+
+### Bounds and constraints
+
+A typevar's bounds and constraints cannot be generic, cyclic or otherwise:
+
+```py
+from typing import Any, TypeVar
+
+S = TypeVar("S")
+
+# TODO: error
+T = TypeVar("T", bound=list[S])
+
+# TODO: error
+U = TypeVar("U", list["T"])
+
+# TODO: error
+V = TypeVar("V", list[S], str)
+
+# TODO: error
+W = TypeVar("W", list["W"], str)
+```
+
+However, they are lazily evaluated and can cyclically refer to their own type:
+
+```py
+from typing import TypeVar, Generic
+
+T = TypeVar("T", bound=list["G"])
+
+class G(Generic[T]):
+    x: T
+
+reveal_type(G[list[G]]().x)  # revealed: list[G[Unknown]]
+```
+
+### Defaults
+
+Defaults can be generic, but can only refer to earlier typevars:
+
+```py
+from typing import Generic, TypeVar
+
+T = TypeVar("T")
+U = TypeVar("U", default=T)
+
+class C(Generic[T, U]):
+    x: T
+    y: U
+
+reveal_type(C[int, str]().x)  # revealed: int
+reveal_type(C[int, str]().y)  # revealed: str
+reveal_type(C[int]().x)  # revealed: int
+reveal_type(C[int]().y)  # revealed: int
+
+# TODO: error
+V = TypeVar("V", default="V")
+
+class D(Generic[V]):
+    x: V
+
+reveal_type(D().x)  # revealed: V@D
+```
+
 [generics]: https://typing.python.org/en/latest/spec/generics.html

crates/ty_python_semantic/src/ast_node_ref.rs

@@ -30,7 +30,7 @@ use ruff_text_size::Ranged;
 /// This means that changes to expressions in other scopes don't invalidate the expression's id, giving
 /// us some form of scope-stable identity for expressions. Only queries accessing the node field
 /// run on every AST change. All other queries only run when the expression's identity changes.
-#[derive(Clone)]
+#[derive(Clone, PartialEq)]
 pub struct AstNodeRef<T> {
     /// The index of the node in the AST.
     index: NodeIndex,
@@ -40,15 +40,14 @@ pub struct AstNodeRef<T> {
     kind: ruff_python_ast::NodeKind,
     #[cfg(debug_assertions)]
     range: ruff_text_size::TextRange,
-    // Note that because the module address is not stored in release builds, `AstNodeRef`
-    // cannot implement `Eq`, as indices are only unique within a given instance of the
-    // AST.
     #[cfg(debug_assertions)]
     module_addr: usize,
 
     _node: PhantomData<T>,
 }
 
+impl<T> Eq for AstNodeRef<T> where T: PartialEq {}
+
 impl<T> AstNodeRef<T> {
     pub(crate) fn index(&self) -> NodeIndex {
         self.index

crates/ty_python_semantic/src/semantic_index/definition.rs

@@ -706,13 +706,6 @@ impl DefinitionKind<'_> {
         matches!(self, DefinitionKind::Assignment(_))
     }
 
-    pub(crate) fn as_typevar(&self) -> Option<&AstNodeRef<ast::TypeParamTypeVar>> {
-        match self {
-            DefinitionKind::TypeVar(type_var) => Some(type_var),
-            _ => None,
-        }
-    }
-
     /// Returns the [`TextRange`] of the definition target.
     ///
     /// A definition target would mainly be the node representing the place being defined i.e.,

crates/ty_python_semantic/src/types.rs

@@ -33,7 +33,7 @@ pub(crate) use self::subclass_of::{SubclassOfInner, SubclassOfType};
 use crate::module_name::ModuleName;
 use crate::module_resolver::{KnownModule, resolve_module};
 use crate::place::{Boundness, Place, PlaceAndQualifiers, imported_symbol};
-use crate::semantic_index::definition::Definition;
+use crate::semantic_index::definition::{Definition, DefinitionKind};
 use crate::semantic_index::place::ScopedPlaceId;
 use crate::semantic_index::scope::ScopeId;
 use crate::semantic_index::{imported_modules, place_table, semantic_index};
@@ -4514,17 +4514,17 @@ impl<'db> Type<'db> {
                                 Parameter::positional_or_keyword(Name::new_static("name"))
                                     .with_annotated_type(Type::LiteralString),
                                 Parameter::variadic(Name::new_static("constraints"))
-                                    .type_form()
+                                    .deferred_type_form()
                                     .with_annotated_type(Type::any()),
                                 Parameter::keyword_only(Name::new_static("bound"))
-                                    .type_form()
+                                    .deferred_type_form()
                                     .with_annotated_type(UnionType::from_elements(
                                         db,
                                         [Type::any(), Type::none(db)],
                                     ))
                                     .with_default_type(Type::none(db)),
                                 Parameter::keyword_only(Name::new_static("default"))
-                                    .type_form()
+                                    .deferred_type_form()
                                     .with_annotated_type(Type::any())
                                     .with_default_type(KnownClass::NoneType.to_instance(db)),
                                 Parameter::keyword_only(Name::new_static("contravariant"))
@@ -7775,35 +7775,87 @@ impl<'db> TypeVarInstance<'db> {
         ))
     }
 
-    #[salsa::tracked(cycle_fn=lazy_bound_cycle_recover, cycle_initial=lazy_bound_cycle_initial)]
+    #[salsa::tracked(cycle_fn=lazy_bound_cycle_recover, cycle_initial=lazy_bound_cycle_initial, heap_size=ruff_memory_usage::heap_size)]
     fn lazy_bound(self, db: &'db dyn Db) -> Option<TypeVarBoundOrConstraints<'db>> {
         let definition = self.definition(db)?;
         let module = parsed_module(db, definition.file(db)).load(db);
-        let typevar_node = definition.kind(db).as_typevar()?.node(&module);
-        let ty = definition_expression_type(db, definition, typevar_node.bound.as_ref()?);
+        let ty = match definition.kind(db) {
+            // PEP 695 typevar
+            DefinitionKind::TypeVar(typevar) => {
+                let typevar_node = typevar.node(&module);
+                Some(definition_expression_type(
+                    db,
+                    definition,
+                    typevar_node.bound.as_ref()?,
+                ))
+            }
+            // legacy typevar
+            DefinitionKind::Assignment(assignment) => {
+                let call_expr = assignment.value(&module).as_call_expr()?;
+                let expr = &call_expr.arguments.find_keyword("bound")?.value;
+                Some(definition_expression_type(db, definition, expr))
+            }
+            _ => None,
+        }?;
         Some(TypeVarBoundOrConstraints::UpperBound(ty))
     }
 
-    #[salsa::tracked]
+    #[salsa::tracked(heap_size=ruff_memory_usage::heap_size)]
     fn lazy_constraints(self, db: &'db dyn Db) -> Option<TypeVarBoundOrConstraints<'db>> {
         let definition = self.definition(db)?;
         let module = parsed_module(db, definition.file(db)).load(db);
-        let typevar_node = definition.kind(db).as_typevar()?.node(&module);
-        let ty = definition_expression_type(db, definition, typevar_node.bound.as_ref()?)
-            .into_union()?;
+        let ty = match definition.kind(db) {
+            // PEP 695 typevar
+            DefinitionKind::TypeVar(typevar) => {
+                let typevar_node = typevar.node(&module);
+                Some(
+                    definition_expression_type(db, definition, typevar_node.bound.as_ref()?)
+                        .into_union()?,
+                )
+            }
+            // legacy typevar
+            DefinitionKind::Assignment(assignment) => {
+                let call_expr = assignment.value(&module).as_call_expr()?;
+                Some(
+                    UnionType::from_elements(
+                        db,
+                        call_expr
+                            .arguments
+                            .args
+                            .iter()
+                            .skip(1)
+                            .map(|arg| definition_expression_type(db, definition, arg)),
+                    )
+                    .into_union()?,
+                )
+            }
+            _ => None,
+        }?;
         Some(TypeVarBoundOrConstraints::Constraints(ty))
     }
 
-    #[salsa::tracked]
+    #[salsa::tracked(heap_size=ruff_memory_usage::heap_size)]
     fn lazy_default(self, db: &'db dyn Db) -> Option<Type<'db>> {
         let definition = self.definition(db)?;
         let module = parsed_module(db, definition.file(db)).load(db);
-        let typevar_node = definition.kind(db).as_typevar()?.node(&module);
-        Some(definition_expression_type(
-            db,
-            definition,
-            typevar_node.default.as_ref()?,
-        ))
+        match definition.kind(db) {
+            // PEP 695 typevar
+            DefinitionKind::TypeVar(typevar) => {
+                let typevar_node = typevar.node(&module);
+                Some(definition_expression_type(
+                    db,
+                    definition,
+                    typevar_node.default.as_ref()?,
+                ))
+            }
+            // legacy typevar
+            DefinitionKind::Assignment(assignment) => {
+                let call_expr = assignment.value(&module).as_call_expr()?;
+                let expr = &call_expr.arguments.find_keyword("default")?.value;
+                Some(definition_expression_type(db, definition, expr))
+            }
+            _ => None,
+        }
     }
 }
 

crates/ty_python_semantic/src/types/class.rs

@@ -29,9 +29,9 @@ use crate::types::{
     DataclassParams, DeprecatedInstance, FindLegacyTypeVarsVisitor, HasRelationToVisitor,
     IsEquivalentVisitor, KnownInstanceType, ManualPEP695TypeAliasType, MaterializationKind,
     NormalizedVisitor, PropertyInstanceType, StringLiteralType, TypeAliasType, TypeContext,
-    TypeMapping, TypeRelation, TypeVarBoundOrConstraints, TypeVarInstance, TypeVarKind,
-    TypedDictParams, UnionBuilder, VarianceInferable, declaration_type, determine_upper_bound,
-    infer_definition_types,
+    TypeMapping, TypeRelation, TypeVarBoundOrConstraintsEvaluation, TypeVarDefaultEvaluation,
+    TypeVarInstance, TypeVarKind, TypedDictParams, UnionBuilder, VarianceInferable,
+    declaration_type, determine_upper_bound, infer_definition_types,
 };
 use crate::{
     Db, FxIndexMap, FxOrderSet, Program,
@@ -4908,7 +4908,6 @@ impl KnownClass {
         context: &InferContext<'db, '_>,
         index: &SemanticIndex<'db>,
         overload: &mut Binding<'db>,
-        call_arguments: &CallArguments<'_, 'db>,
         call_expression: &ast::ExprCall,
     ) {
         let db = context.db();
@@ -5113,26 +5112,9 @@ impl KnownClass {
                 }
 
                 let bound_or_constraint = match (bound, constraints) {
-                    (Some(bound), None) => {
-                        Some(TypeVarBoundOrConstraints::UpperBound(*bound).into())
-                    }
+                    (Some(_), None) => Some(TypeVarBoundOrConstraintsEvaluation::LazyUpperBound),
 
-                    (None, Some(_constraints)) => {
-                        // We don't use UnionType::from_elements or UnionBuilder here,
-                        // because we don't want to simplify the list of constraints like
-                        // we do with the elements of an actual union type.
-                        // TODO: Consider using a new `OneOfType` connective here instead,
-                        // since that more accurately represents the actual semantics of
-                        // typevar constraints.
-                        let elements = UnionType::new(
-                            db,
-                            overload
-                                .arguments_for_parameter(call_arguments, 1)
-                                .map(|(_, ty)| ty)
-                                .collect::<Box<_>>(),
-                        );
-                        Some(TypeVarBoundOrConstraints::Constraints(elements).into())
-                    }
+                    (None, Some(_)) => Some(TypeVarBoundOrConstraintsEvaluation::LazyConstraints),
 
                     // TODO: Emit a diagnostic that TypeVar cannot be both bounded and
                     // constrained
@@ -5149,7 +5131,7 @@ impl KnownClass {
                         Some(containing_assignment),
                         bound_or_constraint,
                         Some(variance),
-                        default.map(Into::into),
+                        default.map(|_| TypeVarDefaultEvaluation::Lazy),
                         TypeVarKind::Legacy,
                     ),
                 )));

crates/ty_python_semantic/src/types/infer.rs

@@ -44,6 +44,7 @@ use salsa;
 use salsa::plumbing::AsId;
 
 use crate::Db;
+use crate::ast_node_ref::AstNodeRef;
 use crate::semantic_index::ast_ids::node_key::ExpressionNodeKey;
 use crate::semantic_index::definition::Definition;
 use crate::semantic_index::expression::Expression;
@@ -631,9 +632,12 @@ struct DefinitionInferenceExtra<'db> {
     /// Is this a cycle-recovery inference result, and if so, what kind?
     cycle_recovery: Option<CycleRecovery<'db>>,
 
-    /// The definitions that are deferred.
+    /// The definitions that have some deferred parts.
     deferred: Box<[Definition<'db>]>,
 
+    /// Deferred expressions within a deferred definition.
+    deferred_expressions: Box<[AstNodeRef<ast::Expr>]>,
+
     /// The diagnostics for this region.
     diagnostics: TypeCheckDiagnostics,