Merge pull request #2312 from manyfold3d/rearrange-workers

Improve behaviour of background workers

Procfile

@@ -1,2 +1,3 @@
 rails: bundle exec rails server -p $PORT
-worker: bundle exec sidekiq
+default_worker: bundle exec sidekiq -C config/workers/default.yml
+performance_worker: bundle exec sidekiq -C config/workers/performance.yml

Procfile.dev

@@ -1,5 +1,6 @@
 rails: SCOUT_DEV_TRACE=true bin/rails server -p $PORT
-worker: bundle exec sidekiq
+default_worker: bundle exec sidekiq -C config/workers/default.yml
+performance_worker: bundle exec sidekiq -C config/workers/performance.yml
 js: yarn build --watch
 css: yarn build:css --watch
 typecheck: yarn typecheck --watch --preserveWatchOutput

app/jobs/analysis/file_conversion_job.rb

@@ -7,7 +7,8 @@ class NonManifoldError < StandardError
 end
 
 class Analysis::FileConversionJob < ApplicationJob
- queue_as :analysis
+ queue_as :performance
+ sidekiq_options retry: false
 
  def perform(file_id, output_format)
  # Can we output this format?

app/jobs/analysis/geometric_analysis_job.rb

@@ -2,7 +2,8 @@ class MeshLoadError < StandardError
 end
 
 class Analysis::GeometricAnalysisJob < ApplicationJob
- queue_as :analysis
+ queue_as :performance
+ sidekiq_options retry: false
 
  def perform(file_id)
  # Get model

app/jobs/application_job.rb

@@ -1,12 +1,9 @@
 class ApplicationJob < ActiveJob::Base
  include ActiveJob::Status
- sidekiq_options retry: false
-
- # Automatically retry jobs that encountered a deadlock
- # retry_on ActiveRecord::Deadlocked
+ sidekiq_options retry: 10
 
  # Most jobs are safe to ignore if the underlying records are no longer available
- # discard_on ActiveJob::DeserializationError
+ discard_on ActiveJob::DeserializationError
 
  before_perform do |job|
  begin

config/sidekiq.yml → config/workers/default.yml

@@ -1,8 +1,9 @@
 ---
+:concurrency: <%= ENV.fetch("DEFAULT_WORKER_CONCURRENCY", 4) %>
 :queues:
  - scan
- - default
  - analysis
+ - default
 :scheduler:
  :dynamic: true
  :dynamic_every: 15s

config/workers/performance.yml

@@ -0,0 +1,4 @@
+---
+:concurrency: <%= ENV.fetch("PERFORMANCE_WORKER_CONCURRENCY", 1) %>
+:queues:
+ - performance