Denormalize Status Message to that Entry Look Up Can Be Fast (#913)

* Thats it, Ive had enough. we need status_message as a column on entry. Today is the day

* call the cops

* call the cops

* Add guard condition for column existence

---------

Co-authored-by: Jeremy Friesen <jeremy.n.friesen@gmail.com>

app/models/bulkrax/importer.rb

@@ -123,16 +123,16 @@ def last_run
       @last_run ||= self.importer_runs.last
     end
 
+    def failed_entries?
+      entries.failed.any?
+    end
+
     def failed_statuses
       @failed_statuses ||= Bulkrax::Status.latest_by_statusable
                                           .includes(:statusable)
                                           .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Failed')
     end
 
-    def failed_entries
-      @failed_entries ||= failed_statuses.map(&:statusable)
-    end
-
     def failed_messages
       failed_statuses.each_with_object({}) do |e, i|
         i[e.error_message] ||= []
@@ -146,10 +146,6 @@ def completed_statuses
                                              .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Complete')
     end
 
-    def completed_entries
-      @completed_entries ||= completed_statuses.map(&:statusable)
-    end
-
     def seen
       @seen ||= {}
     end

app/models/bulkrax/status.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class Status < ApplicationRecord
-    belongs_to :statusable, polymorphic: true
+    belongs_to :statusable, polymorphic: true, denormalize: { fields: %i[status_message], if: :latest? }
     belongs_to :runnable, polymorphic: true
     serialize :error_backtrace, Array
 
@@ -21,5 +21,9 @@ def self.latest_by_statusable_subtable
       status_table.join(latest_status_query.as(latest_status_table.name.to_s), Arel::Nodes::InnerJoin)
                   .on(status_table[:id].eq(latest_status_table[:latest_status_id]))
     end
+
+    def latest?
+      self.id == self.class.where(statusable_id: self.statusable_id, statusable_type: self.statusable_type).order('id desc').pick(:id)
+    end
   end
 end

app/models/concerns/bulkrax/status_info.rb

@@ -10,6 +10,9 @@ module StatusInfo
               as: :statusable,
               class_name: "Bulkrax::Status",
               inverse_of: :statusable
+      scope :failed, -> { where(status_message: 'Failed') }
+      scope :complete, -> { where(status_message: 'Complete') }
+      scope :pending, -> { where(status_message: 'Pending') }
     end
 
     def current_status

bulkrax.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'rails', '>= 5.1.6'
   s.add_dependency 'bagit', '~> 0.4'
   s.add_dependency 'coderay'
+  s.add_dependency 'denormalize_fields'
   s.add_dependency 'iso8601', '~> 0.9.0'
   s.add_dependency 'kaminari'
   s.add_dependency 'language_list', '~> 1.2', '>= 1.2.1'

db/migrate/20240208005801_denormalize_status_message.rb

@@ -0,0 +1,7 @@
+class DenormalizeStatusMessage < ActiveRecord::Migration[5.2]
+  def change
+    add_column :bulkrax_entries, :status_message, :string, default: 'Pending' unless column_exists?(:bulkrax_entries, :status_message)
+    add_column :bulkrax_importers, :status_message, :string, default: 'Pending' unless column_exists?(:bulkrax_importers, :status_message)
+    add_column :bulkrax_exporters, :status_message, :string, default: 'Pending' unless column_exists?(:bulkrax_exporters, :status_message)
+  end
+end

lib/bulkrax.rb

@@ -3,6 +3,7 @@
 require "bulkrax/version"
 require "bulkrax/engine"
 require 'active_support/all'
+require 'denormalize_fields'
 
 # rubocop:disable Metrics/ModuleLength
 module Bulkrax

lib/tasks/bulkrax_tasks.rake

@@ -1,6 +1,18 @@
 # frozen_string_literal: true
 
 namespace :bulkrax do
+  desc 'Update all status messages from the latest status. This is to refresh the denormalized field'
+  task update_status_messages: :environment do
+    @progress = ProgressBar.create(total: Bulkrax::Status.latest_by_statusable.count,
+                                   format: "%a %b\u{15E7}%i %c/%C %p%% %t",
+                                   progress_mark: ' ',
+                                   remainder_mark: "\u{FF65}")
+    Bulkrax::Status.latest_by_statusable.includes(:statusable).find_each do |status|
+      status.statusable.update(status_message: status.status_message)
+      @progress.increment
+    end
+  end
+
   # Usage example: rails bulkrax:generate_test_csvs['5','100','GenericWork']
   desc 'Generate CSVs with fake data for testing purposes'
   task :generate_test_csvs, [:num_of_csvs, :csv_rows, :record_type] => :environment do |_t, args|