🚧 wip: upgrade bulkrax to the latest

Gemfile

@@ -21,8 +21,7 @@ if ENV['DEPENDENCIES_NEXT'] && !ENV['DEPENDENCIES_NEXT'].empty?
   gem 'twitter-typeahead-rails', '0.11.1.pre.corejavascript'
   group :bulkrax do
     # our custom changes require us to lock in the version of bulkrax
-    gem 'bulkrax', git: 'https://github.com/samvera-labs/bulkrax.git', branch: 'gbh-patch'
-    gem 'willow_sword', git: 'https://github.com/notch8/willow_sword.git'
+    gem 'bulkrax', git: 'https://github.com/samvera-labs/bulkrax.git', branch: 'hyrax-4-support'
   end
   gem 'sidekiq', '~> 6.5.6'
 else

Gemfile_next.lock

@@ -8,32 +8,25 @@ GIT
       faraday (~> 1.3.0)
       faraday_middleware
 
-GIT
-  remote: https://github.com/notch8/willow_sword.git
-  revision: 74f7684ff9ca96251f341e50a3afc34b5bd312cc
-  specs:
-    willow_sword (0.2.0)
-      bagit (~> 0.4.1)
-      rails (>= 5.1.6)
-      rubyzip (>= 1.0.0)
-
 GIT
   remote: https://github.com/samvera-labs/bulkrax.git
-  revision: eca45757eabde6fe2e4ad53048c7e639950ea950
-  branch: gbh-patch
+  revision: 28875a8ddc0b1bf8265dbd88fb830ed29a7af570
+  branch: hyrax-4-support
   specs:
-    bulkrax (1.0.2)
+    bulkrax (5.3.0)
       bagit (~> 0.4)
       coderay
+      dry-monads (~> 1.5.0)
       iso8601 (~> 0.9.0)
       kaminari
       language_list (~> 1.2, >= 1.2.1)
-      libxml-ruby (~> 3.1.0)
+      libxml-ruby (~> 3.2.4)
       loofah (>= 2.2.3)
       oai (>= 0.4, < 2.x)
       rack (>= 2.0.6)
       rails (>= 5.1.6)
       rdf (>= 2.0.2, < 4.0)
+      rubyzip
       simple_form
 
 GIT
@@ -591,7 +584,7 @@ GEM
       multi_json
     letter_opener (1.8.1)
       launchy (>= 2.2, < 3)
-    libxml-ruby (3.1.0)
+    libxml-ruby (3.2.4)
     link_header (0.0.8)
     linkeddata (3.2.2)
       json-ld (~> 3.2, >= 3.2.5)
@@ -1193,7 +1186,6 @@ DEPENDENCIES
   webdrivers (~> 4.0)
   webmock (~> 3.7)
   webpacker
-  willow_sword!
 
 BUNDLED WITH
    2.3.7

app/jobs/bulkrax/child_relationships_job_decorator.rb

@@ -50,4 +50,4 @@ def reschedule(entry_id, child_entry_ids, importer_run_id, attempts)
   end
 end
 
-::Bulkrax::ChildRelationshipsJob.prepend(Bulkrax::ChildRelationshipsJobDecorator)
+::Bulkrax::ChildRelationshipsJob.prepend(Bulkrax::ChildRelationshipsJobDecorator) if App.rails_5_1?

app/models/ability.rb

@@ -82,4 +82,16 @@ def ams_aapb_admin_permissions
                                         Contribution,
                                         Annotation ]
   end
+
+  def can_import_works?
+    can_create_any_work?
+  end
+
+  def can_export_works?
+    can_create_any_work?
+  end
+
+  def can_create_any_work?
+    true
+  end
 end

app/models/bulkrax/csv_entry.rb

@@ -18,10 +18,6 @@ def self.data_for_entry(data, _source_id = nil)
     # model has to be separated so that it doesn't get mistranslated by to_h
     raw_data = data.to_h
     raw_data[:model] = data[:model] if data[:model].present?
-    # If the collection field mapping is not 'collection', add 'collection' - the parser needs it
-    raw_data[:collection] = raw_data[collection_field.to_sym] if raw_data.keys.include?(collection_field.to_sym) && collection_field != 'collection'
-    # If the children field mapping is not 'children', add 'children' - the parser needs it
-    raw_data[:children] = raw_data[collection_field.to_sym] if raw_data.keys.include?(children_field.to_sym) && children_field != 'children'
     return raw_data
   end
 

app/views_rails_6_1/bulkrax/entries/show.html.erb → app/views_rails_6_1/bulkrax/entries/show.html.old.erb

@@ -1,4 +1,5 @@
 <%# OVERRIDE BULKRAX 1.0.2 to display ID instead of record class name %>
+<% rails 'hel' %>
 <p id="notice"><%= notice %></p>
 <div class="card">
   <div class="card-body">

config/initializers/bulkrax.rb

@@ -34,6 +34,12 @@ def headers(term = nil)
 
     config.fill_in_blank_source_identifiers = ->(type, index, key_count) { "#{type}-#{index}-#{key_count}" }
 
+    config.field_mappings['Bulkrax::CsvParser'] = {
+      'bulkrax_identifier' => { from: ['source_identifier'], source_identifier: true },
+      'children' => { from: ['children'], split: /\s*[;|]\s*/, related_children_field_mapping: true },
+      'parents' => { from: ['parents'], split: /\s*[;|]\s*/, related_parents_field_mapping: true }
+    }
+
     config.field_mappings['CsvParser'] = {
       'admin_data_gid' => { from: headers('admin_data_gid') },
       'affiliation' => { from: 'Contributor.affiliation' },
@@ -203,12 +209,25 @@ def headers(term = nil)
 
     # Properties that should not be used in imports/exports. They are reserved for use by Hyrax.
     # config.reserved_properties += ['my_field']
+
+    # List of Questioning Authority properties that are controlled via YAML files in
+    # the config/authorities/ directory. For example, the :rights_statement property
+    # is controlled by the active terms in config/authorities/rights_statements.yml
+    # Defaults: 'rights_statement' and 'license'
+    # config.qa_controlled_properties += ['my_field']
+
+    # Specify the delimiter regular expression for splitting an attribute's values into a multi-value array.
+    # config.multi_value_element_split_on = /\s*[:;|]\s*/.freeze
+
+    # Specify the delimiter for joining an attribute's multi-value array into a string.  Note: the
+    # specific delimeter should likely be present in the multi_value_element_split_on expression.
+    # config.multi_value_element_join_on = ' | '
   end
 end
 # rubocop:enable Metrics/BlockLength
 
 # # Sidebar for hyrax 3+ support
-#   if Object.const_defined?(:Hyrax) && ::Hyrax::DashboardController&.respond_to?(:sidebar_partials)
-#     path = "hyrax/dashboard/sidebar/bulkrax_sidebar_additions"
-#     Hyrax::DashboardController.sidebar_partials[:repository_content] << path
-#   end
+  # if Object.const_defined?(:Hyrax) && ::Hyrax::DashboardController&.respond_to?(:sidebar_partials)
+  #   path = "hyrax/dashboard/sidebar/bulkrax_sidebar_additions"
+  #   Hyrax::DashboardController.sidebar_partials[:repository_content] << path
+  # end

db/migrate/20230824174822_rename_children_counters_to_relationships.bulkrax.rb

@@ -0,0 +1,7 @@
+# This migration comes from bulkrax (originally 20211203195233)
+class RenameChildrenCountersToRelationships < ActiveRecord::Migration[5.1]
+  def change
+    rename_column :bulkrax_importer_runs, :processed_children, :processed_relationships unless column_exists?(:bulkrax_importer_runs, :processed_relationships)
+    rename_column :bulkrax_importer_runs, :failed_children, :failed_relationships unless column_exists?(:bulkrax_importer_runs, :failed_relationships)
+  end
+end

db/migrate/20230824174823_add_file_set_counters_to_importer_runs.bulkrax.rb

@@ -0,0 +1,8 @@
+# This migration comes from bulkrax (originally 20211220195027)
+class AddFileSetCountersToImporterRuns < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_importer_runs, :processed_file_sets, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_file_sets)
+    add_column :bulkrax_importer_runs, :failed_file_sets, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :failed_file_sets)
+    add_column :bulkrax_importer_runs, :total_file_set_entries, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :total_file_set_entries)
+  end
+end

db/migrate/20230824174824_add_import_attempts_to_entries.bulkrax.rb

@@ -0,0 +1,6 @@
+# This migration comes from bulkrax (originally 20220118001339)
+class AddImportAttemptsToEntries < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_entries, :import_attempts, :integer, default: 0 unless column_exists?(:bulkrax_entries, :import_attempts)
+  end
+end

db/migrate/20230824174825_add_work_counters_to_importer_runs.bulkrax.rb

@@ -0,0 +1,7 @@
+# This migration comes from bulkrax (originally 20220119213325)
+class AddWorkCountersToImporterRuns < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_importer_runs, :processed_works, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_works)
+    add_column :bulkrax_importer_runs, :failed_works, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :failed_works)
+  end
+end

db/migrate/20230824174826_create_bulkrax_pending_relationships.bulkrax.rb

@@ -0,0 +1,14 @@
+# This migration comes from bulkrax (originally 20220301001839)
+class CreateBulkraxPendingRelationships < ActiveRecord::Migration[5.1]
+  def change
+    unless table_exists?(:bulkrax_pending_relationships)
+      create_table :bulkrax_pending_relationships do |t|
+        t.belongs_to :bulkrax_importer_run, foreign_key: true, null: false
+        t.string :parent_id, null: false
+        t.string :child_id, null: false
+
+        t.timestamps
+      end
+    end
+  end
+end

db/migrate/20230824174827_add_order_to_bulkrax_pending_relationships.bulkrax.rb

@@ -0,0 +1,6 @@
+# This migration comes from bulkrax (originally 20220303212810)
+class AddOrderToBulkraxPendingRelationships < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_pending_relationships, :order, :integer, default: 0 unless column_exists?(:bulkrax_pending_relationships, :order)
+  end
+end

db/migrate/20230824174828_add_include_thumbnails_to_bulkrax_exporters.bulkrax.rb

@@ -0,0 +1,6 @@
+# This migration comes from bulkrax (originally 20220412233954)
+class AddIncludeThumbnailsToBulkraxExporters < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_exporters, :include_thumbnails, :boolean, default: false unless column_exists?(:bulkrax_exporters, :include_thumbnails)
+  end
+end

db/migrate/20230824174829_add_generated_metadata_to_bulkrax_exporters.bulkrax.rb

@@ -0,0 +1,6 @@
+# This migration comes from bulkrax (originally 20220413180915)
+class AddGeneratedMetadataToBulkraxExporters < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_exporters, :generated_metadata, :boolean, default: false unless column_exists?(:bulkrax_exporters, :generated_metadata)
+  end
+end

db/migrate/20230824174830_rename_bulkrax_importer_run_to_importer_run.bulkrax.rb

@@ -0,0 +1,18 @@
+# This migration comes from bulkrax (originally 20220609001128)
+class RenameBulkraxImporterRunToImporterRun < ActiveRecord::Migration[5.1]
+  def up
+    if column_exists?(:bulkrax_pending_relationships, :bulkrax_importer_run_id)
+      remove_foreign_key :bulkrax_pending_relationships, :bulkrax_importer_runs
+      remove_index :bulkrax_pending_relationships, column: :bulkrax_importer_run_id
+
+      rename_column :bulkrax_pending_relationships, :bulkrax_importer_run_id, :importer_run_id
+
+      add_foreign_key :bulkrax_pending_relationships, :bulkrax_importer_runs, column: :importer_run_id
+      add_index :bulkrax_pending_relationships, :importer_run_id, name: 'index_bulkrax_pending_relationships_on_importer_run_id'
+    end
+  end
+
+  def down
+    rename_column :bulkrax_pending_relationships, :importer_run_id, :bulkrax_importer_run_id
+  end
+end

db/migrate/20230824174831_add_indices_to_bulkrax.bulkrax.rb

@@ -0,0 +1,17 @@
+# This migration comes from bulkrax (originally 20230608153601)
+class AddIndicesToBulkrax < ActiveRecord::Migration[5.1]
+  def change
+    check_and_add_index :bulkrax_entries, :identifier
+    check_and_add_index :bulkrax_entries, :type
+    check_and_add_index :bulkrax_entries, [:importerexporter_id, :importerexporter_type], name: 'bulkrax_entries_importerexporter_idx'
+    check_and_add_index :bulkrax_pending_relationships, :child_id
+    check_and_add_index :bulkrax_pending_relationships, :parent_id
+    check_and_add_index :bulkrax_statuses, :error_class
+    check_and_add_index :bulkrax_statuses, [:runnable_id, :runnable_type], name: 'bulkrax_statuses_runnable_idx'
+    check_and_add_index :bulkrax_statuses, [:statusable_id, :statusable_type], name: 'bulkrax_statuses_statusable_idx'
+  end
+
+  def check_and_add_index(table_name, column_name, options = {})
+    add_index(table_name, column_name, options) unless index_exists?(table_name, column_name, options)
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20211214001152) do
+ActiveRecord::Schema.define(version: 20230824174831) do
 
   create_table "admin_data", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci" do |t|
     t.text "sonyci_id"
@@ -62,7 +62,11 @@
     t.datetime "updated_at", null: false
     t.datetime "last_error_at"
     t.datetime "last_succeeded_at"
+    t.integer "import_attempts", default: 0
+    t.index ["identifier"], name: "index_bulkrax_entries_on_identifier"
+    t.index ["importerexporter_id", "importerexporter_type"], name: "bulkrax_entries_importerexporter_idx"
     t.index ["importerexporter_id"], name: "index_bulkrax_entries_on_importerexporter_id"
+    t.index ["type"], name: "index_bulkrax_entries_on_type"
   end
 
   create_table "bulkrax_exporter_runs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci" do |t|
@@ -93,6 +97,8 @@
     t.date "finish_date"
     t.string "work_visibility"
     t.string "workflow_status"
+    t.boolean "include_thumbnails", default: false
+    t.boolean "generated_metadata", default: false
     t.index ["user_id"], name: "index_bulkrax_exporters_on_user_id"
   end
 
@@ -108,9 +114,14 @@
     t.integer "processed_collections", default: 0
     t.integer "failed_collections", default: 0
     t.integer "total_collection_entries", default: 0
-    t.integer "processed_children", default: 0
-    t.integer "failed_children", default: 0
+    t.integer "processed_relationships", default: 0
+    t.integer "failed_relationships", default: 0
     t.text "invalid_records", limit: 16777215
+    t.integer "processed_file_sets", default: 0
+    t.integer "failed_file_sets", default: 0
+    t.integer "total_file_set_entries", default: 0
+    t.integer "processed_works", default: 0
+    t.integer "failed_works", default: 0
     t.index ["importer_id"], name: "index_bulkrax_importer_runs_on_importer_id"
   end
 
@@ -131,6 +142,18 @@
     t.index ["user_id"], name: "index_bulkrax_importers_on_user_id"
   end
 
+  create_table "bulkrax_pending_relationships", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci" do |t|
+    t.bigint "importer_run_id", null: false
+    t.string "parent_id", null: false
+    t.string "child_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.integer "order", default: 0
+    t.index ["child_id"], name: "index_bulkrax_pending_relationships_on_child_id"
+    t.index ["importer_run_id"], name: "index_bulkrax_pending_relationships_on_importer_run_id"
+    t.index ["parent_id"], name: "index_bulkrax_pending_relationships_on_parent_id"
+  end
+
   create_table "bulkrax_statuses", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci" do |t|
     t.string "status_message"
     t.string "error_class"
@@ -142,6 +165,9 @@
     t.string "runnable_type"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["error_class"], name: "index_bulkrax_statuses_on_error_class"
+    t.index ["runnable_id", "runnable_type"], name: "bulkrax_statuses_runnable_idx"
+    t.index ["statusable_id", "statusable_type"], name: "bulkrax_statuses_statusable_idx"
   end
 
   create_table "checksum_audit_logs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci" do |t|
@@ -753,6 +779,7 @@
   add_foreign_key "annotations", "admin_data", column: "admin_data_id"
   add_foreign_key "bulkrax_exporter_runs", "bulkrax_exporters", column: "exporter_id"
   add_foreign_key "bulkrax_importer_runs", "bulkrax_importers", column: "importer_id"
+  add_foreign_key "bulkrax_pending_relationships", "bulkrax_importer_runs", column: "importer_run_id"
   add_foreign_key "collection_type_participants", "hyrax_collection_types"
   add_foreign_key "curation_concerns_operations", "users"
   add_foreign_key "hyrax_batch_ingest_batch_items", "hyrax_batch_ingest_batches", column: "batch_id"