diff --git a/Gemfile b/Gemfile index 2ab90dc..836bf71 100644 --- a/Gemfile +++ b/Gemfile @@ -1,22 +1,27 @@ source 'https://rubygems.org' +puppet_version = ENV['PUPPET_GEM_VERSION'] ||= '6.3.0' + gem 'sinatra' gem 'sinatra-contrib' # gem 'sinatra-authentication' -gem 'activerecord', '~> 4.2', '>= 4.2.6', require: 'active_record' +gem 'activerecord', '4.2.11', require: 'active_record' gem 'bcrypt' gem 'github_changelog_generator' gem 'mcollective-client' gem 'pry' gem 'puma' +gem 'puppet', puppet_version +gem 'puppet_forge', '2.2.8' gem 'r10k' gem 'rake' gem 'require_all' gem 'rocket-chat-notifier' gem 'shotgun' +gem 'sidekiq' gem 'sinatra-activerecord', require: 'sinatra/activerecord' gem 'slack-notifier' -gem 'sqlite3' +gem 'sqlite3', '1.3.13' gem 'thin' gem 'tux' gem 'warden' diff --git a/Gemfile.lock b/Gemfile.lock index 270b2f2..e4a2b08 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -24,11 +24,11 @@ GEM ansi (1.5.0) arel (6.0.4) ast (2.4.0) - backports (3.11.4) + backports (3.12.0) bcrypt (3.1.12) bond (0.5.1) builder (3.2.3) - capybara (3.12.0) + capybara (3.15.0) addressable mini_mime (>= 0.1.3) nokogiri (~> 1.8) @@ -38,7 +38,8 @@ GEM xpath (~> 3.2) coderay (1.1.2) colored (1.2) - concurrent-ruby (1.1.4) + concurrent-ruby (1.1.5) + connection_pool (2.2.2) coveralls (0.8.22) json (>= 1.8, < 3) simplecov (~> 0.16.1) @@ -47,7 +48,7 @@ GEM tins (~> 1.6) crack (0.4.3) safe_yaml (~> 1.0.0) - cri (2.15.2) + cri (2.15.3) colored (~> 1.2) daemons (1.3.1) debase (0.2.2) @@ -56,11 +57,12 @@ GEM diff-lcs (1.3) docile (1.3.1) eventmachine (1.2.7) - faraday (0.13.1) + facter (2.5.1) + faraday (0.9.2) multipart-post (>= 1.2, < 3) faraday-http-cache (2.0.0) faraday (~> 0.8) - faraday_middleware (0.12.2) + faraday_middleware (0.10.1) faraday (>= 0.7.4, < 1.0) fast_gettext (1.1.2) gettext (3.2.9) @@ -79,11 +81,14 @@ GEM rake (>= 10.0) retriable (~> 2.1) hashdiff (0.3.8) + hiera (3.5.0) hirb (0.7.3) + hocon (1.2.5) + httpclient (2.8.3) i18n (0.9.5) concurrent-ruby (~> 1.0) - jaro_winkler (1.5.1) - json (2.1.0) + jaro_winkler (1.5.2) + json (2.2.0) locale (2.1.2) log4r (1.1.10) mcollective-client (2.12.4) @@ -93,31 +98,42 @@ GEM method_source (0.9.2) mini_mime (1.0.1) mini_portile2 (2.4.0) - minitar (0.7) + minitar (0.8) minitest (5.11.3) multi_json (1.13.1) multipart-post (2.0.0) mustermann (1.0.3) - nokogiri (1.9.1) + nokogiri (1.10.2) mini_portile2 (~> 2.4.0) octokit (4.13.0) sawyer (~> 0.8.0, >= 0.5.3) - parallel (1.12.1) - parser (2.5.3.0) + parallel (1.15.0) + parser (2.6.2.0) ast (~> 2.4.0) - powerpack (0.1.2) pry (0.12.2) coderay (~> 1.1.0) method_source (~> 0.9.0) + psych (3.1.0) public_suffix (3.0.3) - puma (3.12.0) - puppet_forge (2.2.9) - faraday (>= 0.9.0, < 0.14.0) - faraday_middleware (>= 0.9.0, < 0.13.0) + puma (3.12.1) + puppet (6.3.0) + facter (> 2.0.1, < 4) + fast_gettext (~> 1.1.2) + hiera (>= 3.2.1, < 4) + httpclient (~> 2.8) + locale (~> 2.1) + multi_json (~> 1.10) + puppet-resource_api (~> 1.5) + semantic_puppet (~> 1.0) + puppet-resource_api (1.8.1) + hocon (>= 1.0) + puppet_forge (2.2.8) + faraday (~> 0.9.0) + faraday_middleware (>= 0.9.0, < 0.11.0) gettext-setup (~> 0.11) minitar semantic_puppet (~> 1.0) - r10k (3.1.0) + r10k (3.1.1) colored (= 1.2) cri (~> 2.15.1) gettext-setup (~> 0.24) @@ -131,6 +147,7 @@ GEM rack (>= 1.0, < 3) rainbow (3.0.0) rake (12.3.2) + redis (4.1.0) regexp_parser (1.3.0) require_all (2.0.0) retriable (2.1.0) @@ -156,24 +173,29 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.8.0) rspec-support (3.8.0) - rubocop (0.61.1) + rubocop (0.66.0) jaro_winkler (~> 1.5.1) parallel (~> 1.10) parser (>= 2.5, != 2.5.1.1) - powerpack (~> 0.1) + psych (>= 3.1.0) rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) - unicode-display_width (~> 1.4.0) + unicode-display_width (>= 1.4.0, < 1.6) ruby-debug-ide (0.6.1) rake (>= 0.8.1) ruby-progressbar (1.10.0) - safe_yaml (1.0.4) + safe_yaml (1.0.5) sawyer (0.8.1) addressable (>= 2.3.5, < 2.6) faraday (~> 0.8, < 1.0) semantic_puppet (1.0.2) shotgun (0.9.2) rack (>= 1.0) + sidekiq (5.2.5) + connection_pool (~> 2.2, >= 2.2.2) + rack (>= 1.5.0) + rack-protection (>= 1.5.0) + redis (>= 3.3.5, < 5) simplecov (0.16.1) docile (~> 1.1) json (>= 1.8, < 3) @@ -220,22 +242,22 @@ GEM sinatra (>= 1.2.1) tzinfo (1.2.5) thread_safe (~> 0.1) - unicode-display_width (1.4.1) - warden (1.2.7) - rack (>= 1.0) + unicode-display_width (1.5.0) + warden (1.2.8) + rack (>= 2.0.6) webmock (3.5.1) addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff xpath (3.2.0) nokogiri (~> 1.8) - yard (0.9.16) + yard (0.9.18) PLATFORMS ruby DEPENDENCIES - activerecord (~> 4.2, >= 4.2.6) + activerecord (= 4.2.11) bcrypt capybara coveralls @@ -245,6 +267,8 @@ DEPENDENCIES mcollective-client pry puma + puppet (= 6.3.0) + puppet_forge (= 2.2.8) r10k rack-test rake @@ -254,13 +278,14 @@ DEPENDENCIES rubocop ruby-debug-ide shotgun + sidekiq simplecov simplecov-console sinatra sinatra-activerecord sinatra-contrib slack-notifier - sqlite3 + sqlite3 (= 1.3.13) thin tux warden diff --git a/app/controllers/api/v1/r10k/environment_controller.rb b/app/controllers/api/v1/r10k/environment_controller.rb index b06b5a3..268eb89 100644 --- a/app/controllers/api/v1/r10k/environment_controller.rb +++ b/app/controllers/api/v1/r10k/environment_controller.rb @@ -10,6 +10,7 @@ class EnvironmentController < ApplicationController prefix = get_prefix(data) branch = get_branch(data) env = get_env(branch, prefix) + paths = APP_CONFIG.module_paths ||= [] EnvironmentController.helpers R10kHelpers @@ -22,7 +23,7 @@ class EnvironmentController < ApplicationController else logger.info("Deploying environment #{env}") # Replace this with Sidekiq - Process.detach(fork { deploy_environment(env, data.deleted) }) + Deploy::EnvironmentWorker.perform_async(branch, config, paths, data.deleted) end end diff --git a/app/controllers/api/v1/r10k/module_controller.rb b/app/controllers/api/v1/r10k/module_controller.rb index 002880d..5b02961 100644 --- a/app/controllers/api/v1/r10k/module_controller.rb +++ b/app/controllers/api/v1/r10k/module_controller.rb @@ -22,7 +22,7 @@ class ModuleController < ApplicationController module_name = sanitize_input(module_name) logger.info("Deploying module #{module_name}") - Process.detach(fork { deploy_module(module_name) }) + Deploy::ModuleWorker.perform_async(module_name, config) end private diff --git a/app/helpers/puppet_helpers.rb b/app/helpers/puppet_helpers.rb new file mode 100644 index 0000000..c599f73 --- /dev/null +++ b/app/helpers/puppet_helpers.rb @@ -0,0 +1,55 @@ +require 'puppet/generate/type' + +# Public methods for executing Puppet methods via the Puppet API. +module PuppetHelpers + # Return an array of modulepaths for the Puppet environment. + # + # @api public + # + # @param name [String] Name of the environment + # @param paths [Array] Array of absolute paths. + # + # @return Array + def self.mod_paths(name, paths) + return paths unless paths.empty? + + ["/etc/puppetlabs/code/environments/#{name}/modules"] + end + + # Create new environment object that will be used for the Puppet Generate types. + # + # @api public + # + # @param name [String] Name of the environment + # @param paths [Array] Array of module path(s) for the environment + # + # @return Puppet::Node::Environment + def self.set_environment(name, paths = []) + module_path = PuppetHelpers.mod_paths(name, paths) + + Puppet::Node::Environment.create(name, module_path) + end + + # Find the inputs from the environment's modulepath + # + # @api public + # + # @param environment [Puppet::Node::Environment] A Puppet environment object. + # + # @return Array[Puppet::Generate::Type::Input] + def self.find_inputs(environment) + Puppet::Generate::Type.find_inputs(:pcore, environment) + end + + # Generate type files from puppet inputs and place them in the output directory. + # + # @api public + # + # @param inputs [Array[Puppet::Generate::Inputs]] Array of Puppet inputs + # @param output [String] Output directory for the generated types. + def self.gen_types(inputs, output) + FileUtils.mkdir(output) unless File.directory?(output) + types = Puppet::Generate::Type.generate(inputs, output) + logger.info(types.message) unless types.nil? + end +end diff --git a/app/helpers/r10k_helpers.rb b/app/helpers/r10k_helpers.rb index da12eb5..77f3a1d 100644 --- a/app/helpers/r10k_helpers.rb +++ b/app/helpers/r10k_helpers.rb @@ -1,76 +1,7 @@ # R10k helper methods for the sinatra app module R10kHelpers - def deploy_environment(branch, deleted) - r10k_args = ['environment', branch] - generate_types = true unless deleted - deploy(r10k_args, generate_types) - end - - def deploy_module(module_name) - r10k_args = ['module', module_name] - generate_types = false - deploy(r10k_args, generate_types) - end - - private - - def deploy(r10k_args, generate_types) - if APP_CONFIG.use_mcollective - results = PuppetWebhook::Orchestrators::MCollective.new('r10k', - 'deploy', - { - dtimeout: APP_CONFIG.discovery_timeout, - timeout: APP_CONFIG.client_timeout - }, - APP_CONFIG.client_cfg, - r10k_args[0].to_sym => r10k_args[1]).run - results.each do |result| - raise result.results[:statusmsg] unless result.results[:statuscode].zero? - end - raise results.stats[:noresponsefrom] unless result.stats[:noresponsefrom].length.zero? - - message = result.results[:statusmsg] - else - command = "#{APP_CONFIG.command_prefix} r10k deploy #{r10k_args[0]} #{r10k_args[1]} #{r10k_arguments(r10k_args[0])}" - message = run_command(command) - end - status_message = { status: :success, message: message.to_s, branch: branch, status_code: 202 } - logger.info("message: #{message} branch: #{branch}") - if generate_types - generate_types(r10k_args[1]) if APP_CONFIG.generate_types - end - notification(status_message) - [status_message[:status_code], status_message.to_json] - rescue StandardError => e - status_message = { status: :fail, message: e.message, trace: e.backtrace, status_code: 500 } - logger.error("message: #{e.message} trace: #{e.backtrace}") - status 500 - notification(status_message) - status_message.to_json - end - - def r10k_arguments(command) - return APP_CONFIG.r10k_deploy_arguments if command == 'environment' - - '-v' - end - - def run_command(command) - message = "forked: #{command}" - system "#{command} &" - message - end - - def generate_types(environment) - command = "#{settings.command_prefix} /opt/puppetlabs/puppet/bin/puppet generate types --environment #{environment}" - - message = run_command(command) - logger.info("message: #{message} environment: #{environment}") - status_message = { status: :success, message: message.to_s, environment: environment, status_code: 200 } - notification(status_message) - rescue StandardError => e - logger.error("message: #{e.message} trace: #{e.backtrace}") - status_message = { status: :fail, message: e.message, trace: e.backtrace, environment: environment, status_code: 500 } - notification(status_message) + def config + config = APP_CONFIG.configfile ||= '/etc/puppetlabs/r10k/r10k.yaml' + YAML.load_file(config) end end diff --git a/app/workers/application_worker.rb b/app/workers/application_worker.rb new file mode 100644 index 0000000..557cd4a --- /dev/null +++ b/app/workers/application_worker.rb @@ -0,0 +1,6 @@ +require 'sidekiq' + +# Main ApplicationWorker class that all other workers inherit from. +class ApplicationWorker + include Sidekiq::Worker +end diff --git a/app/workers/deploy/environment.rb b/app/workers/deploy/environment.rb new file mode 100644 index 0000000..9a24a6a --- /dev/null +++ b/app/workers/deploy/environment.rb @@ -0,0 +1,29 @@ +require 'r10k/action/deploy/environment' +require './app/helpers/puppet_helpers' + +module Deploy + # This worker will deploy a puppet environment using r10k then generate + # puppet resource types from the code in the moduleapth. + class EnvironmentWorker < ApplicationWorker + def perform(environment, config, paths, deleted) + # r10k doesn't like stringified hash keys. This forces symbolized keys + config = config.symbolize_keys + @basedir = config[:sources].first[1]['basedir'] + deploy = R10K::Action::Deploy::Environment.new({ config: config }, [environment], config).call + + if deploy # rubocop:disable Style/GuardClause + generate_types(environment, paths) unless deleted + else + raise 'Failed to deploy' + end + end + + private + + def generate_types(environment, paths) + env_object = PuppetHelpers.set_environment(environment, paths) + inputs = PuppetHelpers.find_inputs(env_object) + PuppetHelpers.gen_types(inputs, "#{@basedir}/#{environment}/.resource_types") + end + end +end diff --git a/app/workers/deploy/module.rb b/app/workers/deploy/module.rb new file mode 100644 index 0000000..f4998ec --- /dev/null +++ b/app/workers/deploy/module.rb @@ -0,0 +1,14 @@ +require 'r10k/action/deploy/module' + +module Deploy + # This worker deploy a puppet module via the R10K API. + class ModuleWorker < ApplicationWorker + def perform(ppt_module, config) + # r10k doesn't like stringified hash keys. This forces symbolized keys + config = config.symbolize_keys + deploy = R10K::Action::Deploy::Module.new({ config: config }, [ppt_module], config).call + + raise 'Failed to deploy' unless deploy + end + end +end diff --git a/config.ru b/config.ru index b27e9d6..84c7149 100644 --- a/config.ru +++ b/config.ru @@ -1,8 +1,16 @@ require './config/environment' +require 'sidekiq/web' raise 'Migrations are pending. Run `rake db:migrate` to resolve the issue.' if ActiveRecord::Migrator.needs_migration? -run ApplicationController -use AuthenticationController -use Api::V1::R10K::EnvironmentController -use Api::V1::R10K::ModuleController +map '/' do + use AuthenticationController + use Api::V1::R10K::EnvironmentController + use Api::V1::R10K::ModuleController + + run ApplicationController +end + +map '/sidekiq' do + run Sidekiq::Web +end diff --git a/config/environment.rb b/config/environment.rb index 4342ad7..f4f1e18 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -3,8 +3,8 @@ require 'bundler/setup' Bundler.require(:default, ENV['SINATRA_ENV']) -config = YAML.load_file('config/config.yml').freeze -APP_CONFIG = OpenStruct.new(config[ENV['SINATRA_ENV']]).freeze +config = YAML.load_file('config/config.yml') +APP_CONFIG = OpenStruct.new(config[ENV['SINATRA_ENV']]) ActiveRecord::Base.establish_connection( adapter: 'sqlite3', diff --git a/config/sidekiq.yml b/config/sidekiq.yml new file mode 100644 index 0000000..91e0a5a --- /dev/null +++ b/config/sidekiq.yml @@ -0,0 +1,8 @@ +--- +:concurrency: 5 +development: + :concurrency: 5 +staging: + :concurrency: 10 +production: + :concurrency: 40 diff --git a/spec/app/helpers/r10k_helpers_spec.rb b/spec/app/helpers/r10k_helpers_spec.rb index 4cafa36..91fc228 100644 --- a/spec/app/helpers/r10k_helpers_spec.rb +++ b/spec/app/helpers/r10k_helpers_spec.rb @@ -7,35 +7,22 @@ class TestHelpers describe R10kHelpers do subject { TestHelpers.new } - describe '#deploy_environment' do - context 'with valid branch' do - let(:branch) { 'master' } - let(:r10k_args) { ['environment', branch] } - let(:deleted) { false } - it 'returns a success' do - allow(subject).to receive(:deploy).with(r10k_args, true).and_return(true) - expect(subject.deploy_environment(branch, deleted)).to eq(true) - end - end - - context 'with invalid branch' do - let(:branch) { false } - let(:r10k_args) { ['environment', branch] } - let(:deleted) { false } - it 'returns a failure' do - allow(subject).to receive(:deploy).with(r10k_args, true).and_return(false) - expect(subject.deploy_environment(branch, deleted)).to eq(false) + describe '#config' do + context 'without configfile' do + let(:r10k_config) do + { + sources: { + main: { + remote: 'git@example.com:foo/bar.git', + basedir: '/etc/foo/bar' + } + } + } end - end - end - describe '#deploy_module' do - context 'with valid module' do - let(:module_name) { 'foo-bar' } - let(:r10k_args) { ['module', module_name] } it 'returns a success' do - allow(subject).to receive(:deploy).with(r10k_args, false).and_return(true) - expect(subject.deploy_module(module_name)).to eq(true) + allow(subject).to receive(:config).and_return(r10k_config) + expect(subject.config).to eq(r10k_config) end end end