Use Mutex#owned? to correctly check if the Mutex is owned by the current Thread or Fiber

Changelog.md

@@ -1,3 +1,10 @@
+### Development
+
+Bug Fixes:
+
+* Use `Mutex#owned?` to allow `RSpec::Support::ReentrantMutex` to work in
+  nested Fibers on Ruby 3.0 and later. (Benoit Daloze, #503, #504)
+
 ### 3.10.2 / 2021-01-28
 [Full Changelog](http://github.com/rspec/rspec-support/compare/v3.10.1...v3.10.2)
 

lib/rspec/support/reentrant_mutex.rb

@@ -27,17 +27,34 @@ def synchronize
 
     private
 
-      def enter
-        @mutex.lock if @owner != Thread.current
-        @owner = Thread.current
-        @count += 1
-      end
+      # This is fixing a bug #501 that is specific to Ruby 3.0. The new implementation
+      # depends on `owned?` that was introduced in Ruby 2.0, so both should work for Ruby 2.x.
+      if RUBY_VERSION.to_f >= 3.0
+        def enter
+          @mutex.lock unless @mutex.owned?
+          @count += 1
+        end
 
-      def exit
-        @count -= 1
-        return unless @count == 0
-        @owner = nil
-        @mutex.unlock
+        def exit
+          unless @mutex.owned?
+            raise ThreadError, "Attempt to unlock a mutex which is locked by another thread/fiber"
+          end
+          @count -= 1
+          @mutex.unlock if @count == 0
+        end
+      else
+        def enter
+          @mutex.lock if @owner != Thread.current
+          @owner = Thread.current
+          @count += 1
+        end
+
+        def exit
+          @count -= 1
+          return unless @count == 0
+          @owner = nil
+          @mutex.unlock
+        end
       end
     end
 

spec/rspec/support/reentrant_mutex_spec.rb

@@ -27,4 +27,28 @@
     mutex.synchronize { order.pass_to :thread, :resume_on => :sleep }
     order.join_all
   end
+
+  if RUBY_VERSION >= '3.0'
+    it 'waits when trying to lock from another Fiber' do
+      mutex.synchronize do
+        ready = false
+        f = Fiber.new do
+          expect {
+            ready = true
+            mutex.send(:enter)
+            raise 'should reach here: mutex is already locked on different Fiber'
+          }.to raise_error(Exception, 'waited correctly')
+        end
+
+        main_thread = Thread.current
+
+        t = Thread.new do
+          Thread.pass until ready && main_thread.stop?
+          main_thread.raise Exception, 'waited correctly'
+        end
+        f.resume
+        t.join
+      end
+    end
+  end
 end