Add a ScopeKind for the __class__ cell

Summary
--

This PR aims to resolve (or help to resolve) #18442 and #19357 by encoding the
CPython semantics around the `__class__` cell in our semantic model. Namely,

> __class__ is an implicit closure reference created by the compiler if any methods in a class body refer to either __class__ or super.

from the Python [docs](https://docs.python.org/3/reference/datamodel.html#creating-the-class-object).

As noted in the code comment by @AlexWaygood, we don't fully model this
behavior, opting always to create the `__class__` cell binding in a new
`ScopeKind::ClassCell` around each method definition, without checking if any
method in the class body actually refers to `__class__` or `super`.

As such, this PR may only help with #18442 and not #19357.

Test Plan
--

Existing tests, plus (TODO) tests from #19783 and #19424, which tackled the
issues above on a per-rule basis.

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>

Author: ntBre

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -703,7 +703,10 @@ impl SemanticSyntaxContext for Checker<'_> {
             match scope.kind {
                 ScopeKind::Class(_) | ScopeKind::Lambda(_) => return false,
                 ScopeKind::Function(ast::StmtFunctionDef { is_async, .. }) => return *is_async,
-                ScopeKind::Generator { .. } | ScopeKind::Module | ScopeKind::Type => {}
+                ScopeKind::Generator { .. }
+                | ScopeKind::Module
+                | ScopeKind::Type
+                | ScopeKind::ClassCell => {}
             }
         }
         false
@@ -714,7 +717,10 @@ impl SemanticSyntaxContext for Checker<'_> {
             match scope.kind {
                 ScopeKind::Class(_) => return false,
                 ScopeKind::Function(_) | ScopeKind::Lambda(_) => return true,
-                ScopeKind::Generator { .. } | ScopeKind::Module | ScopeKind::Type => {}
+                ScopeKind::Generator { .. }
+                | ScopeKind::Module
+                | ScopeKind::Type
+                | ScopeKind::ClassCell => {}
             }
         }
         false
@@ -725,7 +731,7 @@ impl SemanticSyntaxContext for Checker<'_> {
             match scope.kind {
                 ScopeKind::Class(_) | ScopeKind::Generator { .. } => return false,
                 ScopeKind::Function(_) | ScopeKind::Lambda(_) => return true,
-                ScopeKind::Module | ScopeKind::Type => {}
+                ScopeKind::Module | ScopeKind::Type | ScopeKind::ClassCell => {}
             }
         }
         false
@@ -1092,6 +1098,60 @@ impl<'a> Visitor<'a> for Checker<'a> {
                     }
                 }
 
+                // Here we add the implicit scope surrounding a method which allows
+                // code in the method to access `__class__` at runtime. The closure sits
+                // in between the class scope and the function scope.
+                //
+                // Parameter defaults in methods cannot access `__class__`:
+                //
+                // ```pycon
+                // >>> class Bar:
+                // ...     def method(self, x=__class__): ...
+                // ...
+                // Traceback (most recent call last):
+                // File "<python-input-6>", line 1, in <module>
+                //     class Bar:
+                //         def method(self, x=__class__): ...
+                // File "<python-input-6>", line 2, in Bar
+                //     def method(self, x=__class__): ...
+                //                     ^^^^^^^^^
+                // NameError: name '__class__' is not defined
+                // ```
+                //
+                // However, type parameters in methods *can* access `__class__`:
+                //
+                // ```pycon
+                // >>> class Foo:
+                // ...     def bar[T: __class__](): ...
+                // ...
+                // >>> Foo.bar.__type_params__[0].__bound__
+                // <class '__main__.Foo'>
+                // ```
+                //
+                // Note that this is still not 100% accurate! At runtime, the implicit `__class__`
+                // closure is only added if the name `super` (has to be a name -- `builtins.super`
+                // and similar don't count!) or the name `__class__` is used in any method of the
+                // class. However, accurately emulating that would be both complex and probably
+                // quite expensive unless we moved to a double-traversal of each scope similar to
+                // ty. It would also only matter in extreme and unlikely edge cases. So we ignore
+                // that subtlety for now.
+                //
+                // See <https://docs.python.org/3/reference/datamodel.html#creating-the-class-object>.
+                let added_dunder_class_scope = if self.semantic.current_scope().kind.is_class() {
+                    self.semantic.push_scope(ScopeKind::ClassCell);
+                    let binding_id = self.semantic.push_binding(
+                        TextRange::default(),
+                        BindingKind::ClassCell,
+                        BindingFlags::empty(),
+                    );
+                    self.semantic
+                        .current_scope_mut()
+                        .add("__class__", binding_id);
+                    true
+                } else {
+                    false
+                };
+
                 self.semantic.push_scope(ScopeKind::Type);
 
                 if let Some(type_params) = type_params {
@@ -1155,6 +1215,9 @@ impl<'a> Visitor<'a> for Checker<'a> {
                 self.semantic.pop_scope(); // Function scope
                 self.semantic.pop_definition();
                 self.semantic.pop_scope(); // Type parameter scope
+                if added_dunder_class_scope {
+                    self.semantic.pop_scope(); // `__class__` cell closure scope
+                }
                 self.add_binding(
                     name,
                     stmt.identifier(),

crates/ruff_linter/src/renamer.rs

@@ -354,7 +354,10 @@ impl Renamer {
                 ))
             }
             // Avoid renaming builtins and other "special" bindings.
-            BindingKind::FutureImport | BindingKind::Builtin | BindingKind::Export(_) => None,
+            BindingKind::FutureImport
+            | BindingKind::Builtin
+            | BindingKind::Export(_)
+            | BindingKind::ClassCell => None,
             // By default, replace the binding's name with the target name.
             BindingKind::Annotation
             | BindingKind::Argument

crates/ruff_linter/src/rules/pyflakes/mod.rs

@@ -737,6 +737,7 @@ mod tests {
 
     /// A re-implementation of the Pyflakes test runner.
     /// Note that all tests marked with `#[ignore]` should be considered TODOs.
+    #[track_caller]
     fn flakes(contents: &str, expected: &[Rule]) {
         let contents = dedent(contents);
         let source_type = PySourceType::default();

crates/ruff_linter/src/rules/pylint/rules/non_ascii_name.rs

@@ -73,7 +73,8 @@ pub(crate) fn non_ascii_name(checker: &Checker, binding: &Binding) {
         | BindingKind::SubmoduleImport(_)
         | BindingKind::Deletion
         | BindingKind::ConditionalDeletion(_)
-        | BindingKind::UnboundException(_) => {
+        | BindingKind::UnboundException(_)
+        | BindingKind::ClassCell => {
             return;
         }
     };

crates/ruff_python_semantic/src/binding.rs

@@ -446,11 +446,17 @@ impl Ranged for Binding<'_> {
 /// ID uniquely identifying a [Binding] in a program.
 ///
 /// Using a `u32` to identify [Binding]s should be sufficient because Ruff only supports documents with a
-/// size smaller than or equal to `u32::max`. A document with the size of `u32::max` must have fewer than `u32::max`
+/// size smaller than or equal to `u32::MAX`. A document with the size of `u32::MAX` must have fewer than `u32::MAX`
 /// bindings because bindings must be separated by whitespace (and have an assignment).
 #[newtype_index]
 pub struct BindingId;
 
+impl BindingId {
+    pub const fn dunder_class_id() -> Self {
+        BindingId::from_u32(u32::MAX)
+    }
+}
+
 /// The bindings in a program.
 ///
 /// Bindings are indexed by [`BindingId`]
@@ -672,6 +678,9 @@ pub enum BindingKind<'a> {
     /// Stores the ID of the binding that was shadowed in the enclosing
     /// scope, if any.
     UnboundException(Option<BindingId>),
+
+    /// TODO
+    ClassCell,
 }
 
 bitflags! {

crates/ruff_python_semantic/src/model.rs

@@ -404,22 +404,11 @@ impl<'a> SemanticModel<'a> {
             }
         }
 
-        let mut seen_function = false;
         let mut import_starred = false;
         let mut class_variables_visible = true;
         for (index, scope_id) in self.scopes.ancestor_ids(self.scope_id).enumerate() {
             let scope = &self.scopes[scope_id];
             if scope.kind.is_class() {
-                // Allow usages of `__class__` within methods, e.g.:
-                //
-                // ```python
-                // class Foo:
-                //     def __init__(self):
-                //         print(__class__)
-                // ```
-                if seen_function && matches!(name.id.as_str(), "__class__") {
-                    return ReadResult::ImplicitGlobal;
-                }
                 // Do not allow usages of class symbols unless it is the immediate parent
                 // (excluding type scopes), e.g.:
                 //
@@ -442,7 +431,8 @@ impl<'a> SemanticModel<'a> {
             // Allow class variables to be visible for an additional scope level
             // when a type scope is seen — this covers the type scope present between
             // function and class definitions and their parent class scope.
-            class_variables_visible = scope.kind.is_type() && index == 0;
+            class_variables_visible =
+                (scope.kind.is_type() && index == 0) || (scope.kind.is_class_cell() && index == 1);
 
             if let Some(binding_id) = scope.get(name.id.as_str()) {
                 // Mark the binding as used.
@@ -614,7 +604,6 @@ impl<'a> SemanticModel<'a> {
                 }
             }
 
-            seen_function |= scope.kind.is_function();
             import_starred = import_starred || scope.uses_star_imports();
         }
 
@@ -1353,11 +1342,11 @@ impl<'a> SemanticModel<'a> {
         self.scopes[scope_id].parent
     }
 
-    /// Returns the first parent of the given [`Scope`] that is not of [`ScopeKind::Type`], if any.
+    /// Returns the first parent of the given [`Scope`] that is not of [`ScopeKind::Type`] or [`ScopeKind::ClassCell`], if any.
     pub fn first_non_type_parent_scope(&self, scope: &Scope) -> Option<&Scope<'a>> {
         let mut current_scope = scope;
         while let Some(parent) = self.parent_scope(current_scope) {
-            if parent.kind.is_type() {
+            if matches!(parent.kind, ScopeKind::Type | ScopeKind::ClassCell) {
                 current_scope = parent;
             } else {
                 return Some(parent);
@@ -1366,11 +1355,14 @@ impl<'a> SemanticModel<'a> {
         None
     }
 
-    /// Returns the first parent of the given [`ScopeId`] that is not of [`ScopeKind::Type`], if any.
+    /// Returns the first parent of the given [`ScopeId`] that is not of [`ScopeKind::Type`] or [`ScopeKind::ClassCell`], if any.
     pub fn first_non_type_parent_scope_id(&self, scope_id: ScopeId) -> Option<ScopeId> {
         let mut current_scope_id = scope_id;
         while let Some(parent_id) = self.parent_scope_id(current_scope_id) {
-            if self.scopes[parent_id].kind.is_type() {
+            if matches!(
+                self.scopes[parent_id].kind,
+                ScopeKind::Type | ScopeKind::ClassCell
+            ) {
                 current_scope_id = parent_id;
             } else {
                 return Some(parent_id);

crates/ruff_python_semantic/src/scope.rs

@@ -169,6 +169,7 @@ bitflags! {
 #[derive(Debug, is_macro::Is)]
 pub enum ScopeKind<'a> {
     Class(&'a ast::StmtClassDef),
+    ClassCell,
     Function(&'a ast::StmtFunctionDef),
     Generator {
         kind: GeneratorKind,