From 6a936ff2a7e4783b73e66af9c4e51cc83563bda0 Mon Sep 17 00:00:00 2001 From: James Smith Date: Wed, 26 Jun 2024 16:35:08 +0100 Subject: [PATCH 1/7] create a performance worker with concurrency of 1, just to run geometric analysis jobs so that we don't spam the CPU with 5 at a time --- Procfile | 3 ++- Procfile.dev | 3 ++- app/jobs/analysis/geometric_analysis_job.rb | 2 +- config/{sidekiq.yml => workers/default.yml} | 2 +- config/workers/performance.yml | 4 ++++ 5 files changed, 10 insertions(+), 4 deletions(-) rename config/{sidekiq.yml => workers/default.yml} (100%) create mode 100644 config/workers/performance.yml diff --git a/Procfile b/Procfile index 9ae88ecde..1622f815c 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,3 @@ rails: bundle exec rails server -p $PORT -worker: bundle exec sidekiq +default_worker: bundle exec sidekiq -C config/workers/default.yml +performance_worker: bundle exec sidekiq -C config/workers/performance.yml diff --git a/Procfile.dev b/Procfile.dev index 58e557e2c..98c380499 100644 --- a/Procfile.dev +++ b/Procfile.dev @@ -1,5 +1,6 @@ rails: SCOUT_DEV_TRACE=true bin/rails server -p $PORT -worker: bundle exec sidekiq +default_worker: bundle exec sidekiq -C config/workers/default.yml +performance_worker: bundle exec sidekiq -C config/workers/performance.yml js: yarn build --watch css: yarn build:css --watch typecheck: yarn typecheck --watch --preserveWatchOutput diff --git a/app/jobs/analysis/geometric_analysis_job.rb b/app/jobs/analysis/geometric_analysis_job.rb index 634c4c2ca..c203b5be2 100644 --- a/app/jobs/analysis/geometric_analysis_job.rb +++ b/app/jobs/analysis/geometric_analysis_job.rb @@ -2,7 +2,7 @@ class MeshLoadError < StandardError end class Analysis::GeometricAnalysisJob < ApplicationJob - queue_as :analysis + queue_as :performance def perform(file_id) # Get model diff --git a/config/sidekiq.yml b/config/workers/default.yml similarity index 100% rename from config/sidekiq.yml rename to config/workers/default.yml index b077c436b..bbf0a4a28 100644 --- a/config/sidekiq.yml +++ b/config/workers/default.yml @@ -1,8 +1,8 @@ --- :queues: - scan - - default - analysis + - default :scheduler: :dynamic: true :dynamic_every: 15s diff --git a/config/workers/performance.yml b/config/workers/performance.yml new file mode 100644 index 000000000..7fceed1e3 --- /dev/null +++ b/config/workers/performance.yml @@ -0,0 +1,4 @@ +--- +:concurrency: 1 +:queues: + - performance From 56e97193b009b1bf22eabbee303f38a4b45de357 Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 27 Jun 2024 08:40:07 +0100 Subject: [PATCH 2/7] re-enable sidekiq retries --- app/jobs/application_job.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index 181e83998..c5a6328b5 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,12 +1,9 @@ class ApplicationJob < ActiveJob::Base include ActiveJob::Status - sidekiq_options retry: false - - # Automatically retry jobs that encountered a deadlock - # retry_on ActiveRecord::Deadlocked + sidekiq_options retry: 5 # Most jobs are safe to ignore if the underlying records are no longer available - # discard_on ActiveJob::DeserializationError + discard_on ActiveJob::DeserializationError before_perform do |job| begin From 75153079fe3de8ff3d7fe52733b31781aaeba0d0 Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 27 Jun 2024 08:40:24 +0100 Subject: [PATCH 3/7] don't retry if we get a mesh load error --- app/jobs/analysis/geometric_analysis_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/analysis/geometric_analysis_job.rb b/app/jobs/analysis/geometric_analysis_job.rb index c203b5be2..e94989ce0 100644 --- a/app/jobs/analysis/geometric_analysis_job.rb +++ b/app/jobs/analysis/geometric_analysis_job.rb @@ -3,6 +3,7 @@ class MeshLoadError < StandardError class Analysis::GeometricAnalysisJob < ApplicationJob queue_as :performance + discard_on MeshLoadError def perform(file_id) # Get model From 7a5cb18305f1180db49e6a2ecf5b697471bf1237 Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 27 Jun 2024 08:56:29 +0100 Subject: [PATCH 4/7] add concurrency env vars for default and performance workers --- config/workers/default.yml | 1 + config/workers/performance.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/workers/default.yml b/config/workers/default.yml index bbf0a4a28..9c0052f09 100644 --- a/config/workers/default.yml +++ b/config/workers/default.yml @@ -1,4 +1,5 @@ --- +:concurrency: <%= ENV.fetch("DEFAULT_WORKER_CONCURRENCY", 4) %> :queues: - scan - analysis diff --git a/config/workers/performance.yml b/config/workers/performance.yml index 7fceed1e3..7e1bc730b 100644 --- a/config/workers/performance.yml +++ b/config/workers/performance.yml @@ -1,4 +1,4 @@ --- -:concurrency: 1 +:concurrency: <%= ENV.fetch("PERFORMANCE_WORKER_CONCURRENCY", 1) %> :queues: - performance From 691a1db87096fe5f6d2c53f17cd0e577274d45dd Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 27 Jun 2024 08:56:39 +0100 Subject: [PATCH 5/7] increase sidekiq retries --- app/jobs/application_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index c5a6328b5..0fd1b0899 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,6 +1,6 @@ class ApplicationJob < ActiveJob::Base include ActiveJob::Status - sidekiq_options retry: 5 + sidekiq_options retry: 10 # Most jobs are safe to ignore if the underlying records are no longer available discard_on ActiveJob::DeserializationError From 46229f3eaa50558686619db29163dc39272f47aa Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 27 Jun 2024 09:03:32 +0100 Subject: [PATCH 6/7] move file conversion onto performance queue --- app/jobs/analysis/file_conversion_job.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/jobs/analysis/file_conversion_job.rb b/app/jobs/analysis/file_conversion_job.rb index 0176ef501..623bcdfe9 100644 --- a/app/jobs/analysis/file_conversion_job.rb +++ b/app/jobs/analysis/file_conversion_job.rb @@ -7,7 +7,10 @@ class NonManifoldError < StandardError end class Analysis::FileConversionJob < ApplicationJob - queue_as :analysis + queue_as :performance + + discard_on UnsupportedFormatError + discard_on NonManifoldError def perform(file_id, output_format) # Can we output this format? From ced4cf3592bef5b7b482499b358e2fe772632fea Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 27 Jun 2024 09:34:08 +0100 Subject: [PATCH 7/7] don't discard errors, just don't retry performance jobs --- app/jobs/analysis/file_conversion_job.rb | 4 +--- app/jobs/analysis/geometric_analysis_job.rb | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/jobs/analysis/file_conversion_job.rb b/app/jobs/analysis/file_conversion_job.rb index 623bcdfe9..630e0fd01 100644 --- a/app/jobs/analysis/file_conversion_job.rb +++ b/app/jobs/analysis/file_conversion_job.rb @@ -8,9 +8,7 @@ class NonManifoldError < StandardError class Analysis::FileConversionJob < ApplicationJob queue_as :performance - - discard_on UnsupportedFormatError - discard_on NonManifoldError + sidekiq_options retry: false def perform(file_id, output_format) # Can we output this format? diff --git a/app/jobs/analysis/geometric_analysis_job.rb b/app/jobs/analysis/geometric_analysis_job.rb index e94989ce0..6395e0c83 100644 --- a/app/jobs/analysis/geometric_analysis_job.rb +++ b/app/jobs/analysis/geometric_analysis_job.rb @@ -3,7 +3,7 @@ class MeshLoadError < StandardError class Analysis::GeometricAnalysisJob < ApplicationJob queue_as :performance - discard_on MeshLoadError + sidekiq_options retry: false def perform(file_id) # Get model