ActiveJob concurrency raises FrozenError

[christianrolle]

After I included GoodJob::ActiveJobExtensions::Concurrency into my recurring job:

class SynchronizeMediaJob < ApplicationJob
  include GoodJob::ActiveJobExtensions::Concurrency

  good_job_control_concurrency_with(
    # Maximum number of unfinished jobs to allow with the concurrency key
    total_limit: 1,
    key: -> { 'SynchronizeMediaJob' }
  )

  def perform
    Medium.unpushed.find_each |medium|
      medium.push!
    end
  end
end

and

class ApplicationJob < ActiveJob::Base
  class GatewayConnectException < StandardError; end
  retry_on GatewayConnectException, wait: :exponentially_longer, attempts: 5
end

All my specs are failing due to:

An error occurred while loading ./spec/requests/visits/update_spec.rb.
Failure/Error: require File.expand_path('../config/environment', __dir__)

FrozenError:
  can't modify frozen #<Class:#<Array:0x00005559009bde90>>: ["/usr/src/app/app/channels", "/usr/src/app/app/controllers", "/usr/src/app/app/controllers/concerns", "/usr/src/app/app/datatables", "/usr/src/app/app/decorators", "/usr/src/app/app/forms", "/usr/src/app/app/helpers", "/usr/src/app/app/interactors", "/usr/src/app/app/jobs", "/usr/src/app/app/mailers", "/usr/src/app/app/models", "/usr/src/app/app/models/concerns", "/usr/src/app/app/policies", "/usr/src/app/app/representers", "/usr/src/app/app/services", #<Pathname:/usr/src/app/lib>, "/bundle/ruby/2.7.0/gems/devise-4.7.3/app/controllers", "/bundle/ruby/2.7.0/gems/devise-4.7.3/app/helpers", "/bundle/ruby/2.7.0/gems/devise-4.7.3/app/mailers", "/bundle/ruby/2.7.0/gems/actiontext-6.1.0/app/helpers", "/bundle/ruby/2.7.0/gems/actiontext-6.1.0/app/models", "/bundle/ruby/2.7.0/gems/actionmailbox-6.1.0/app/controllers", "/bundle/ruby/2.7.0/gems/actionmailbox-6.1.0/app/jobs", "/bundle/ruby/2.7.0/gems/actionmailbox-6.1.0/app/models", "/bundle/ruby/2.7.0/gems/activestorage-6.1.0/app/controllers", "/bundle/ruby/2.7.0/gems/activestorage-6.1.0/app/controllers/concerns", "/bundle/ruby/2.7.0/gems/activestorage-6.1.0/app/jobs", "/bundle/ruby/2.7.0/gems/activestorage-6.1.0/app/models"]
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/change_observer.rb:21:in `unshift'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/change_observer.rb:21:in `unshift'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/engine.rb:588:in `block in <class:Engine>'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:32:in `instance_exec'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:32:in `run'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:61:in `block in run_initializers'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:50:in `each'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:50:in `tsort_each_child'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:60:in `run_initializers'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/application.rb:384:in `initialize!'
# ./config/environment.rb:5:in `<top (required)>'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:23:in `require'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:23:in `block in require_with_bootsnap_lfi'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/loaded_features_index.rb:92:in `register'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:22:in `require_with_bootsnap_lfi'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:31:in `require'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/kernel.rb:34:in `require'
# ./spec/rails_helper.rb:4:in `<top (required)>'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:23:in `require'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:23:in `block in require_with_bootsnap_lfi'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/loaded_features_index.rb:92:in `register'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:22:in `require_with_bootsnap_lfi'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:31:in `require'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/kernel.rb:34:in `require'
# ./spec/requests/visits/update_spec.rb:1:in `<top (required)>'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:59:in `load'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:59:in `load'

My GoodJob configuration:

# config/environments/production.rb

require "active_support/core_ext/integer/time"
require Rails.root.join 'app/services/smtp_password_service.rb'

Rails.application.configure do
  config.cache_classes = true
  config.eager_load = true
  config.consider_all_requests_local       = false
  config.action_controller.perform_caching = true
  config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present?
  config.assets.compile = false
  config.active_storage.service = :amazon
  config.log_level = :info
  config.log_tags = [:request_id]
  config.action_mailer.perform_caching = false
  config.action_mailer.deliver_later_queue_name = 'default'
  config.action_mailer.default_options = {
    charset: 'UTF-8'
  }
  config.action_mailer.perform_deliveries = true
  config.action_mailer.smtp_settings = {
    address: ENV['SMTP_HOST'],
    port: 587,
    user_name: ENV['AWS_ACCESS_KEY_ID'],
    password: SmtpPasswordService.build(secret: ENV['AWS_SECRET_ACCESS_KEY'],
                                        region: ENV['AWS_REGION']),
    authentication: 'plain',
    enable_starttls_auto: true
  }
  config.i18n.fallbacks = true
  config.active_support.deprecation = :notify
  config.active_support.disallowed_deprecation = :log
  config.active_support.disallowed_deprecation_warnings = []
  config.log_formatter = ::Logger::Formatter.new
  config.active_record.dump_schema_after_migration = false

  config.good_job.enable_cron = true

  config.good_job.cron = {
    # Every minute
    synchronize_media: {
      cron: '* * * * *',
      class: 'SynchronizeMediaJob',
      set: { priority: -1 },
      description: 'Synchronizes visit media with Gateway'
    }
  }
end

As soon as I comment out the ActiveJobExtensions::Concurrency related, the specs are running.
Any idea?

[bensheldon]

@christianrolle I haven't seen that error before and I'm happy to help figure out what's happening. From looking at StackOverflow, the frozen array error might not be the root cause: https://stackoverflow.com/a/50371654/241735

Could you please try targeting a single spec (e.g. bundle exec rspec spec/requests/visits/update_spec.rb) and share the complete output. Also, could you share what is in your config/environments/test.rb file?

[christianrolle]

@bensheldon The config/environments/test.rb:

require "active_support/core_ext/integer/time"

# The test environment is used exclusively to run your application's
# test suite. You never need to work with it otherwise. Remember that
# your test database is "scratch space" for the test suite and is wiped
# and recreated between test runs. Don't rely on the data there!

Rails.application.configure do
  # Settings specified here will take precedence over those in config/application.rb.
  config.action_mailer.default_url_options = { host: 'www.example.com' }
  config.cache_classes = false
  config.action_view.cache_template_loading = true

  # Do not eager load code on boot. This avoids loading your whole application
  # just for the purpose of running a single test. If you are using a tool that
  # preloads Rails for running tests, you may have to set it to true.
  # Note: Set to `true` so that we can use SimpleCov effectively.
  config.eager_load = true

  # Configure public file server for tests with Cache-Control for performance.
  config.public_file_server.enabled = true
  config.public_file_server.headers = {
    'Cache-Control' => "public, max-age=#{1.hour.to_i}"
  }

  # Show full error reports and disable caching.
  config.consider_all_requests_local       = true
  config.action_controller.perform_caching = false
  config.cache_store = :null_store

  # Raise exceptions instead of rendering exception templates.
  config.action_dispatch.show_exceptions = false

  # Disable request forgery protection in test environment.
  config.action_controller.allow_forgery_protection = false

  # Store uploaded files on the local file system in a temporary directory.
  config.active_storage.service = :test

  config.action_mailer.perform_caching = false

  # Tell Action Mailer not to deliver emails to the real world.
  # The :test delivery method accumulates sent emails in the
  # ActionMailer::Base.deliveries array.
  config.action_mailer.delivery_method = :test

  # Print deprecation notices to the stderr.
  config.active_support.deprecation = :stderr

  # Raise exceptions for disallowed deprecations.
  config.active_support.disallowed_deprecation = :raise

  # Tell Active Support which deprecation messages to disallow.
  config.active_support.disallowed_deprecation_warnings = []

  # Raises error for missing translations.
  # config.i18n.raise_on_missing_translations = true

  # Annotate rendered view with file names.
  # config.action_view.annotate_rendered_view_with_filenames = true

  config.active_job.queue_adapter = :test

  # Gateway configs
  config.x.gateway.url = 'www.example.com/'
  config.x.gateway.admin_user = 'test_admin'
  config.x.gateway.admin_pass = 'test_admin'
  config.x.gateway.push_user = 'test'
  config.x.gateway.push_pass = 'test'
end

Running bundle exec rspec spec/requests/visits/update_spec.rb:

DEPRECATION WARNING: Initialization autoloaded the constants ApplicationMailer, ActionText::ContentHelper, and ActionText::TagHelper.

Being able to do this is deprecated. Autoloading during initialization is going
to be an error condition in future versions of Rails.

Reloading does not reboot the application, and therefore code executed during
initialization does not run again. So, if you reload ApplicationMailer, for example,
the expected changes won't be reflected in that stale Class object.

These autoloaded constants have been unloaded.

In order to autoload safely at boot time, please wrap your code in a reloader
callback this way:

    Rails.application.reloader.to_prepare do
      # Autoload classes and modules needed at boot time here.
    end

That block runs when the application boots, and every time there is a reload.
For historical reasons, it may run twice, so it has to be idempotent.

Check the "Autoloading and Reloading Constants" guide to learn more about how
Rails autoloads and reloads.
 (called from <top (required)> at /usr/src/app/config/environment.rb:5)

An error occurred while loading ./spec/requests/visits/update_spec.rb.
Failure/Error: include GoodJob::ActiveJobExtensions::Concurrency

NameError:
  uninitialized constant GoodJob::ActiveJobExtensions
# ./app/jobs/synchronize_media_job.rb:3:in `<class:SynchronizeMediaJob>'
# ./app/jobs/synchronize_media_job.rb:2:in `<top (required)>'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:23:in `require'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:23:in `block in require_with_bootsnap_lfi'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/loaded_features_index.rb:92:in `register'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:22:in `require_with_bootsnap_lfi'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:31:in `require'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/kernel.rb:26:in `require'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:409:in `const_get'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:409:in `block (2 levels) in eager_load'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:733:in `block in ls'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:725:in `foreach'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:725:in `ls'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:404:in `block in eager_load'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:393:in `synchronize'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:393:in `eager_load'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:508:in `each'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:508:in `eager_load_all'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/application/finisher.rb:133:in `block in <module:Finisher>'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:32:in `instance_exec'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:32:in `run'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:61:in `block in run_initializers'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:60:in `run_initializers'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/application.rb:384:in `initialize!'
# ./config/environment.rb:5:in `<top (required)>'
# ./spec/rails_helper.rb:4:in `require'
# ./spec/rails_helper.rb:4:in `<top (required)>'
# ./spec/requests/visits/update_spec.rb:1:in `require'
# ./spec/requests/visits/update_spec.rb:1:in `<top (required)>'
No examples found.


Finished in 0.00007 seconds (files took 2.41 seconds to load)
0 examples, 0 failures, 1 error occurred outside of examples

Coverage report generated for RSpec to /usr/src/app/coverage. 440 / 834 LOC (52.76%) covered.
Stopped processing SimpleCov as a previous error not related to SimpleCov has been detected

Furthremore on Rails CLI I did:

GoodJob.constants
=>
[ 0] :CurrentExecution,
    [ 1] :Railtie,
    [ 2] :Adapter,
    [ 3] :Daemon,
    [ 4] :MultiScheduler,
    [ 5] :JobPerformer,
    [ 6] :LogSubscriber,
    [ 7] :Notifier,
    [ 8] :VERSION,
    [ 9] :Poller,
    [10] :Configuration,
    [11] :Scheduler,
    [12] :Lockable,
    [13] :CLI,
    [14] :Job

And there really is no GoodJob::ActiveJobExtensions. Maybe I miss out on some configuration?

[bensheldon]

@christianrolle hmmm. That is strange. What version of GoodJob is in your Gemfile.lock?

Here's my output:

[1] pry(main)> GoodJob.constants
=> [:Poller,
 :Scheduler,
 :Railtie,
 :VERSION,
 :ExecutionResult,
 :LogSubscriber,
 :Job,
 :Execution,
 :Configuration,
 :Daemon,
 :MultiScheduler,
 :CurrentThread,
 :CLI,
 :ActiveJobExtensions,
 :JobPerformer,
 :Adapter,
 :CronEntry,
 :CronManager,
 :Lockable,
 :Notifier]

Also, this may be entirely the wrong direction, but I wonder what the Rails autoloader. is doing to unload constants with those Deprecation warnings (DEPRECATION WARNING: Initialization autoloaded the constants ApplicationMailer, ActionText::ContentHelper, and ActionText::TagHelper.). I had to go through my production Rails apps to track down what those problems were. Here's what I did:

1. Add pp caller_locations to the top of one of the files that is autoloader (e.g. open up the ActionMailer gem and edit the source file; afterwards do gem pristine actionmailer)
2. Run the command that caused the problem DISABLE_SPRING=1 DISABLE_BOOTSNAP=1 ...
3. Look for a reference to config/initializers directory; that's the loader. It might also be in a gem, which isn't so easily fixable.
4. Rewrite the code so that it fits inside of Rails.application.reloader.to_prepare do block

[christianrolle]

@bensheldon
My Gemfile.lock:

good_job (1.8.0)
  activejob (>= 5.2.0)
  activerecord (>= 5.2.0)
  concurrent-ruby (>= 1.0.2)
  railties (>= 5.2.0)
  thor (>= 0.14.1)
  zeitwerk (>= 2.0)

[christianrolle]

@bensheldon I could get rid of the DEPRECATION WARNING. It was indeed a reference to ApplicationMailer in a initializer...
Now running my test looks like:

An error occurred while loading ./spec/requests/visits/update_spec.rb.
Failure/Error: include GoodJob::ActiveJobExtensions::Concurrency

NameError:
  uninitialized constant GoodJob::ActiveJobExtensions
# ./app/jobs/synchronize_media_job.rb:3:in `<class:SynchronizeMediaJob>'
# ./app/jobs/synchronize_media_job.rb:2:in `<top (required)>'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:23:in `require'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:23:in `block in require_with_bootsnap_lfi'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/loaded_features_index.rb:92:in `register'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:22:in `require_with_bootsnap_lfi'
# /bundle/ruby/2.7.0/gems/bootsnap-1.5.1/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:31:in `require'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/kernel.rb:26:in `require'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:409:in `const_get'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:409:in `block (2 levels) in eager_load'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:733:in `block in ls'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:725:in `foreach'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:725:in `ls'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:404:in `block in eager_load'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:393:in `synchronize'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:393:in `eager_load'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:508:in `each'
# /bundle/ruby/2.7.0/gems/zeitwerk-2.4.2/lib/zeitwerk/loader.rb:508:in `eager_load_all'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/application/finisher.rb:133:in `block in <module:Finisher>'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:32:in `instance_exec'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:32:in `run'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:61:in `block in run_initializers'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/initializable.rb:60:in `run_initializers'
# /bundle/ruby/2.7.0/gems/railties-6.1.0/lib/rails/application.rb:384:in `initialize!'
# ./config/environment.rb:5:in `<top (required)>'
# ./spec/rails_helper.rb:4:in `require'
# ./spec/rails_helper.rb:4:in `<top (required)>'
# ./spec/requests/visits/update_spec.rb:1:in `require'
# ./spec/requests/visits/update_spec.rb:1:in `<top (required)>'
No examples found.


Finished in 0.00005 seconds (files took 2.35 seconds to load)
0 examples, 0 failures, 1 error occurred outside of examples

Coverage report generated for RSpec to /usr/src/app/coverage. 438 / 835 LOC (52.46%) covered.
Stopped processing SimpleCov as a previous error not related to SimpleCov has been detected

[bensheldon]

@christianrolle aha! I think I have the answer for you based on the version of GoodJob you're using: The Concurrency extension was added in v1.11.0.

I recommend updating straight to v1.99 though, and from there to the 2.0 branch: https://github.com/bensheldon/good_job#upgrading-minor-versions

[christianrolle]

@bensheldon thank you very much for the hint. I will upgrade to v2.x via v1.9.