Performance regression in CollectionAssocation#build

activerecord/lib/active_record/associations/collection_association.rb

@@ -79,6 +79,7 @@ def ids_writer(ids)
       def reset
         super
         @target = []
+        @replaced_targets = Set.new
         @association_ids = nil
       end
 
@@ -268,10 +269,7 @@ def load_target
       end
 
       def add_to_target(record, skip_callbacks: false, replace: false, &block)
-        if replace || association_scope.distinct_value
-          index = @target.index(record)
-        end
-        replace_on_target(record, index, skip_callbacks, &block)
+        replace_on_target(record, skip_callbacks, replace: replace || association_scope.distinct_value, &block)
       end
 
       def target=(record)
@@ -281,7 +279,7 @@ def target=(record)
         when Array
           super
         else
-          add_to_target(record, skip_callbacks: true, replace: true)
+          replace_on_target(record, true, replace: true, inversing: true)
         end
       end
 
@@ -418,7 +416,7 @@ def replace_common_records_in_memory(new_target, original_target)
           common_records = intersection(new_target, original_target)
           common_records.each do |record|
             skip_callbacks = true
-            replace_on_target(record, @target.index(record), skip_callbacks)
+            replace_on_target(record, skip_callbacks, replace: true)
           end
         end
 
@@ -441,7 +439,11 @@ def concat_records(records, raise = false)
           records
         end
 
-        def replace_on_target(record, index, skip_callbacks)
+        def replace_on_target(record, skip_callbacks, replace:, inversing: false)
+          if replace && (!record.new_record? || @replaced_targets.include?(record))
+            index = @target.index(record)
+          end
+
           catch(:abort) do
             callback(:before_add, record)
           end || return unless skip_callbacks
@@ -452,6 +454,8 @@ def replace_on_target(record, index, skip_callbacks)
 
           yield(record) if block_given?
 
+          @replaced_targets << record if inversing || index
+
           if index
             target[index] = record
           elsif @_was_loaded || !loaded?