From 63c29e1d58d6aab0771983ac2120e5e23ba7ad8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ezr?= Date: Wed, 16 Oct 2019 03:46:51 +0200 Subject: [PATCH] Fixes #28067 - dynflow sidekiq services config --- .fixtures.yml | 1 + manifests/config.pp | 28 ++++++++++++++++++++++++++ manifests/init.pp | 9 +++++++++ manifests/install.pp | 6 ++++++ manifests/params.pp | 3 +++ manifests/service.pp | 24 ++++++++++++++++++++-- metadata.json | 4 ++++ spec/classes/foreman_install_spec.rb | 11 ++++++++++ spec/classes/foreman_service_spec.rb | 11 +++++++++- spec/classes/foreman_spec.rb | 4 +++- templates/dynflow_orchestrator.yml.erb | 3 +++ templates/dynflow_worker.yml.erb | 4 ++++ templates/settings.yaml.erb | 3 +++ 13 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 templates/dynflow_orchestrator.yml.erb create mode 100644 templates/dynflow_worker.yml.erb diff --git a/.fixtures.yml b/.fixtures.yml index d97c1f14a..ed23010f4 100644 --- a/.fixtures.yml +++ b/.fixtures.yml @@ -13,6 +13,7 @@ fixtures: mysql: 'https://github.com/puppetlabs/puppetlabs-mysql' postgresql: 'https://github.com/puppetlabs/puppetlabs-postgresql' puppet: 'https://github.com/theforeman/puppet-puppet' + redis: 'https://github.com/voxpupuli/puppet-redis' systemd: 'https://github.com/camptocamp/puppet-systemd' selinux_core: repo: "https://github.com/puppetlabs/puppetlabs-selinux_core" diff --git a/manifests/config.pp b/manifests/config.pp index 26af09883..4b0ee5290 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -80,6 +80,34 @@ ensure => absent, } + if $::foreman::jobs_sidekiq { + if $::foreman::jobs_sidekiq_redis_url == undef { + include ::redis' + + redis::instance { 'dynflow': + port => $::foreman::jobs_sidekiq_redis_port, + } + } + + file { '/etc/foreman/dynflow/': + ensure => directory, + } + + file { '/etc/foreman/dynflow/orchestrator.yml': + owner => 'root', + group => $::foreman::group, + mode => '0640', + content => template('foreman/dynflow_orchestrator.yml.erb'), + } + + file { '/etc/foreman/dynflow/worker.yml': + owner => 'root', + group => $::foreman::group, + mode => '0640', + content => template('foreman/dynflow_worker.yml.erb'), + } + } + if $::foreman::apache { contain foreman::config::apache diff --git a/manifests/init.pp b/manifests/init.pp index 8c7b7f12c..19a373147 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -204,6 +204,12 @@ # # $jobs_service_ensure:: Whether the Dynflow executor should be running or stopped # +# $jobs_sidekiq:: Whether use Sidekiq to run Dynflow executor +# +# $jobs_sidekiq_redis_url:: If set, the redis server is not managed and we use the defined url to connect +# +# $jobs_sidekiq_redis_port:: What port should redis server for Dynflow worker use +# # $hsts_enabled:: Should HSTS enforcement in https requests be enabled # # $cors_domains:: List of domains that show be allowed for Cross-Origin Resource Sharing. This requires Foreman 1.22+ @@ -305,6 +311,9 @@ String $jobs_service = $::foreman::params::jobs_service, Stdlib::Ensure::Service $jobs_service_ensure = $::foreman::params::jobs_service_ensure, Boolean $jobs_service_enable = $::foreman::params::jobs_service_enable, + Boolean $jobs_sidekiq = $::foreman::params::jobs_sidekiq, + Optional[Redis::RedisUrl] $jobs_sidekiq_redis_url = $::foreman::params::jobs_sidekiq_redis_url, + Stdlib::Port $jobs_sidekiq_redis_port = $::foreman::params::jobs_sidekiq_redis_port, Boolean $hsts_enabled = $::foreman::params::hsts_enabled, Array[Stdlib::HTTPUrl] $cors_domains = $::foreman::params::cors_domains, ) inherits foreman::params { diff --git a/manifests/install.pp b/manifests/install.pp index 0ecde9c1c..32539ae60 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -37,6 +37,12 @@ } } + if $::foreman::jobs_sidekiq { + package { 'foreman-dynflow-sidekiq': + ensure => installed, + } + } + if $::foreman::use_foreman_service { package { 'foreman-service': ensure => installed, diff --git a/manifests/params.pp b/manifests/params.pp index 16a41bbea..e303ee007 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -95,6 +95,9 @@ $jobs_service = 'dynflowd' $jobs_service_ensure = 'running' $jobs_service_enable = true + $jobs_sidekiq = true + $jobs_sidekiq_redis_url = undef + $jobs_sidekiq_redis_port = 6381 $hsts_enabled = true diff --git a/manifests/service.pp b/manifests/service.pp index f18daa939..7a0d9ee44 100644 --- a/manifests/service.pp +++ b/manifests/service.pp @@ -11,10 +11,30 @@ String $jobs_service = $::foreman::jobs_service, Stdlib::Ensure::Service $jobs_service_ensure = $::foreman::jobs_service_ensure, Boolean $jobs_service_enable = $::foreman::jobs_service_enable, + Boolean $jobs_sidekiq = $::foreman::jobs_sidekiq, ) { + if $jobs_sidekiq { + $sidekiq_ensure = $jobs_service_ensure + $sidekiq_enable = $jobs_service_enable + $dynflowd_ensure = 'stopped' + $dynflowd_enable = false + } else { + $dynflowd_ensure = $jobs_service_ensure + $dynflowd_enable = $jobs_service_enable + $sidekiq_enable = false + $sidekiq_ensure = 'stopped' + } + service { 'dynflow-sidekiq@orchestrator': + ensure => $sidekiq_ensure, + enable => $sidekiq_enable, + } + service { 'dynflow-sidekiq@worker': + ensure => $sidekiq_ensure, + enable => $sidekiq_enable, + } service { $jobs_service: - ensure => $jobs_service_ensure, - enable => $jobs_service_enable, + ensure => $dynflowd_ensure, + enable => $dynflowd_enable, } if $apache { diff --git a/metadata.json b/metadata.json index 4003c6c6e..30cd648bf 100644 --- a/metadata.json +++ b/metadata.json @@ -35,6 +35,10 @@ { "name": "puppet/extlib", "version_requirement": ">= 3.0.0 < 5.0.0" + }, + { + "name": "puppet/redis", + "version_requirement": ">= 4.2.0 < 5.0.0" } ], "requirements": [ diff --git a/spec/classes/foreman_install_spec.rb b/spec/classes/foreman_install_spec.rb index cc4d9a83b..1ce20d6c4 100644 --- a/spec/classes/foreman_install_spec.rb +++ b/spec/classes/foreman_install_spec.rb @@ -19,6 +19,17 @@ it { should contain_package('foreman-postgresql').that_requires('Class[foreman::repo]') } end + describe 'sidekiq jobs' do + context 'with jobs_sidekiq disabled' do + let(:params) { super().merge(jobs_sidekiq: false) } + it { is_expected.not_to contain_package('foreman-dynflow-sidekiq') } + end + context 'with jobs_sidekiq enabled' do + let(:params) { super().merge(jobs_sidekiq: true) } + it { is_expected.to contain_package('foreman-dynflow-sidekiq') } + end + end + context 'with SELinux enabled' do let(:facts) { super().merge(selinux: true) } diff --git a/spec/classes/foreman_service_spec.rb b/spec/classes/foreman_service_spec.rb index 26f1c53ea..d62b3440b 100644 --- a/spec/classes/foreman_service_spec.rb +++ b/spec/classes/foreman_service_spec.rb @@ -17,7 +17,8 @@ foreman_service_enable: true, jobs_service: 'dynflower', jobs_service_ensure: 'stopped', - jobs_service_enable: false + jobs_service_enable: false, + jobs_sidekiq: false } end @@ -67,4 +68,12 @@ .with_enable(true) end end + + context 'with sidekiq' do + let(:pre_condition) { 'include ::apache' } + let(:params) { super().merge(jobs_sidekiq: true, jobs_service_ensure: 'running', jobs_service_enable: true) } + it { is_expected.to contain_service('dynflow-sidekiq@orchestrator').with_ensure('running').with_enable(true) } + it { is_expected.to contain_service('dynflow-sidekiq@worker').with_ensure('running').with_enable(true) } + it { is_expected.to contain_service('dynflower').with_ensure('stopped').with_enable(false) } + end end diff --git a/spec/classes/foreman_spec.rb b/spec/classes/foreman_spec.rb index 2c01da8bc..3e940e3ab 100644 --- a/spec/classes/foreman_spec.rb +++ b/spec/classes/foreman_spec.rb @@ -134,7 +134,9 @@ # service it { should contain_class('foreman::service') } it { should_not contain_service('foreman') } - it { is_expected.to contain_service('dynflowd').with_ensure('running').with_enable(true) } + it { is_expected.to contain_service('dynflow-sidekiq@orchestrator').with_ensure('running').with_enable(true) } + it { is_expected.to contain_service('dynflow-sidekiq@worker').with_ensure('running').with_enable(true) } + it { is_expected.to contain_service('dynflowd').with_ensure('stopped').with_enable(false) } it 'should restart passenger' do should contain_exec('restart_foreman') diff --git a/templates/dynflow_orchestrator.yml.erb b/templates/dynflow_orchestrator.yml.erb new file mode 100644 index 000000000..565bf1778 --- /dev/null +++ b/templates/dynflow_orchestrator.yml.erb @@ -0,0 +1,3 @@ +:concurrency: 1 +:queues: + - dynflow_orchestrator diff --git a/templates/dynflow_worker.yml.erb b/templates/dynflow_worker.yml.erb new file mode 100644 index 000000000..a42218fff --- /dev/null +++ b/templates/dynflow_worker.yml.erb @@ -0,0 +1,4 @@ +:concurrency: <%= scope.lookupvar("foreman::dynflow_pool_size") %> +:queues: + - default + - remote_execution diff --git a/templates/settings.yaml.erb b/templates/settings.yaml.erb index 9aa1bd2e2..5b039c3f7 100644 --- a/templates/settings.yaml.erb +++ b/templates/settings.yaml.erb @@ -74,6 +74,9 @@ :dynflow: :pool_size: <%= scope.lookupvar("foreman::dynflow_pool_size") %> +<% if scope.lookupvar("foreman::jobs_sidekiq") %> + :redis_url: <%= scope.lookupvar("foreman::jobs_sidekiq_redis_url") || "redis://localhost:#{scope.lookupvar("foreman::jobs_sidekiq_redis_port")}/0" %> +<% end %> <% if scope.lookupvar("foreman::apache") && !scope.lookupvar("foreman::passenger") -%> # Configure reverse proxy headers