Merge pull request #19383 from lfu/ansible_runner_cleanup

Cleanup after Ansible runner.

(cherry picked from commit 1e1ef97)

https://bugzilla.redhat.com/show_bug.cgi?id=1784103

app/models/manageiq/providers/embedded_ansible/automation_manager/job.rb

@@ -4,7 +4,6 @@ class ManageIQ::Providers::EmbeddedAnsible::AutomationManager::Job < ManageIQ::P
   require_nested :Status
 
   belongs_to :ext_management_system, :foreign_key => :ems_id, :class_name => "ManageIQ::Providers::AutomationManager", :inverse_of => false
-  belongs_to :job_template, :foreign_key => :orchestration_template_id, :class_name => "ConfigurationScript", :inverse_of => false
   belongs_to :playbook, :foreign_key => :configuration_script_base_id, :inverse_of => false
 
   belongs_to :miq_task, :foreign_key => :ems_ref, :inverse_of => false
@@ -16,22 +15,20 @@ class ManageIQ::Providers::EmbeddedAnsible::AutomationManager::Job < ManageIQ::P
   #   :limit      => String
   #   :extra_vars => Hash
   #
-  def self.create_stack(template, options = {})
-    template_ref = template.new_record? ? nil : template
-    new(:name                  => template.name,
-        :ext_management_system => template.manager,
-        :verbosity             => template.variables["verbosity"].to_i,
-        :authentications       => collect_authentications(template.manager, options),
-        :job_template          => template_ref).tap do |stack|
-      stack.send(:update_with_provider_object, raw_create_stack(template, options))
+  def self.create_stack(playbook, options = {})
+    new(:name                  => playbook.name,
+        :ext_management_system => playbook.manager,
+        :verbosity             => options[:verbosity].to_i,
+        :authentications       => collect_authentications(playbook.manager, options),
+        :playbook              => playbook).tap do |stack|
+      stack.send(:update_with_provider_object, raw_create_stack(playbook, options))
     end
   end
 
-  def self.raw_create_stack(template, options = {})
-    options = reconcile_extra_vars_keys(template, options)
-    template.run(options)
+  def self.raw_create_stack(playbook, options = {})
+    playbook.run(options)
   rescue StandardError => e
-    _log.error("Failed to create job from template(#{template.name}), error: #{e}")
+    _log.error("Failed to create job from playbook(#{playbook.name}), error: #{e}")
     raise MiqException::MiqOrchestrationProvisionError, e.to_s, e.backtrace
   end
 
@@ -60,6 +57,10 @@ def job_plays
     resources.where(:resource_category => 'job_play').order(:start_time)
   end
 
+  def playbook_set_stats
+    raw_stdout_json.dig(-1, 'event_data', 'artifact_data')
+  end
+
   # Intend to be called by UI to display stdout. The stdout is stored in MiqTask#task_results or #message if error
   # Since the task_results may contain a large block of data, it is desired to remove the task upon receiving the data
   def raw_stdout_via_worker(userid, format = 'txt')
@@ -93,14 +94,6 @@ def retireable?
 
   private
 
-  # If extra_vars are passed through automate, all keys are considered as attributes and
-  # converted to lower case. Need to convert them back to original definitions in the
-  # job template through survey_spec or variables
-  def self.reconcile_extra_vars_keys(_template, options)
-    options
-  end
-  private_class_method :reconcile_extra_vars_keys
-
   def self.collect_authentications(manager, options)
     credential_ids = options.values_at(
       :credential,

app/models/manageiq/providers/embedded_ansible/automation_manager/playbook.rb

@@ -1,48 +1,48 @@
 class ManageIQ::Providers::EmbeddedAnsible::AutomationManager::Playbook < ManageIQ::Providers::EmbeddedAutomationManager::ConfigurationScriptPayload
   has_many :jobs, :class_name => 'OrchestrationStack', :foreign_key => :configuration_script_base_id
 
-  def run(options, userid = nil)
-    options[:playbook_id] = id
-    options[:userid]      = userid || 'system'
-    options[:name]        = "Playbook: #{name}"
-    miq_job = ManageIQ::Providers::EmbeddedAnsible::AutomationManager::PlaybookRunner.create_job(options)
-    miq_job.signal(:start)
-    miq_job.miq_task.id
-  end
-
-  # return provider raw object
-  def raw_create_job_template(options)
-    job_template_klass = ManageIQ::Providers::EmbeddedAnsible::AutomationManager::ConfigurationScript
-    jt_options = build_parameter_list(options)
-    _log.info("Creating job template with options:")
-    $log.log_hashes(jt_options)
-    job_template_klass.raw_create_in_provider(manager, jt_options)
-  end
+  DEFAULT_EXECUTION_TTL = 100.minutes # automate state machine aborts after 100 retries at a minute interval
 
   def self.display_name(number = 1)
     n_('Playbook (Embedded Ansible)', 'Playbooks (Embedded Ansible)', number)
   end
 
-  private
+  def run(vars = {})
+    workflow = ManageIQ::Providers::AnsiblePlaybookWorkflow
 
-  def build_parameter_list(options)
-    params = {
-      :name                     => options[:template_name] || "#{name}_#{SecureRandom.uuid}",
-      :description              => options[:description] || "Created on #{Time.zone.now}",
-      :project                  => configuration_script_source.manager_ref,
-      :playbook                 => name,
-      :become_enabled           => options[:become_enabled].present?,
-      :verbosity                => options[:verbosity].presence || 0,
-      :limit                    => options[:limit],
-      :inventory                => options[:inventory] || manager.provider.default_inventory,
-      :extra_vars               => options[:extra_vars].try(:to_json)
+    extra_vars = build_extra_vars(vars[:extra_vars])
+
+    playbook_vars = {
+      :configuration_script_source_id => configuration_script_source_id,
+      :playbook_relative_path         => name
     }
 
-    %i(credential vault_credential cloud_credential network_credential).each do |credential|
-      cred_sym = "#{credential}_id".to_sym
-      params[credential] = Authentication.find(options[cred_sym]).native_ref if options[cred_sym].present?
+    credentials = collect_credentials(vars)
+
+    kwargs = {:become_enabled => vars[:become_enabled]}
+    kwargs[:timeout]   = vars[:execution_ttl].present? ? vars[:execution_ttl].to_i.minutes : DEFAULT_EXECUTION_TTL
+    kwargs[:verbosity] = vars[:verbosity].to_i if vars[:verbosity].present?
+
+    workflow.create_job({}, extra_vars, playbook_vars, vars[:hosts], credentials, kwargs).tap do |job|
+      job.signal(:start)
     end
+  end
+
+  private
+
+  def build_extra_vars(external = {})
+    (external || {}).each_with_object({}) do |(k, v), hash|
+      match_data = v.kind_of?(String) && /password::/.match(v)
+      hash[k] = match_data ? ManageIQ::Password.decrypt(v.gsub(/password::/, '')) : v
+    end
+  end
 
-    params.compact
+  def collect_credentials(options)
+    options.values_at(
+      :credential,
+      :cloud_credential,
+      :network_credential,
+      :vault_credential
+    ).compact
   end
 end

app/models/mixins/ansible_playbook_mixin.rb

@@ -0,0 +1,43 @@
+module AnsiblePlaybookMixin
+  extend ActiveSupport::Concern
+
+  CONFIG_OPTIONS_WHITELIST = %i[
+    become_enabled
+    cloud_credential_id
+    credential_id
+    execution_ttl
+    extra_vars
+    hosts
+    network_credential_id
+    vault_credential_id
+    verbosity
+  ].freeze
+
+  def translate_credentials!(options)
+    %i[credential vault_credential network_credential cloud_credential].each do |cred|
+      cred_sym = "#{cred}_id".to_sym
+      credential_id = options.delete(cred_sym)
+      options[cred] = Authentication.find(credential_id).native_ref if credential_id.present?
+    end
+  end
+
+  def use_default_inventory?(hosts)
+    hosts.blank? || hosts == 'localhost'
+  end
+
+  def hosts_array(hosts_string)
+    return ["localhost"] if use_default_inventory?(hosts_string)
+
+    hosts_string.split(',').map(&:strip).delete_blanks
+  end
+
+  def playbook_log_stdout(log_option, job)
+    return unless %(on_error always).include?(log_option)
+    return if log_option == 'on_error' && job.raw_status.succeeded?
+
+    $log.info("Stdout from ansible job #{job.name}: #{job.raw_stdout('txt_download')}")
+  rescue StandardError => err
+    $log.error("Failed to get stdout from ansible job #{job.name}")
+    $log.log_backtrace(err)
+  end
+end

app/models/service_ansible_playbook.rb

@@ -1,7 +1,8 @@
 class ServiceAnsiblePlaybook < ServiceGeneric
   include AnsibleExtraVarsMixin
+  include AnsiblePlaybookMixin
 
-  delegate :job_template, :to => :service_template, :allow_nil => true
+  delegate :playbook, :to => :service_template, :allow_nil => true
 
   # A chance for taking options from automate script to override options from a service dialog
   def preprocess(action, add_options = {})
@@ -37,7 +38,7 @@ def launch_ansible_job_queue(action)
   end
 
   def launch_ansible_job(action)
-    jt = job_template(action)
+    my_playbook = playbook(action)
     opts = get_job_options(action).deep_merge(
       :extra_vars => {
         'manageiq'            => service_manageiq_env(action),
@@ -48,7 +49,7 @@ def launch_ansible_job(action)
 
     _log.info("Launching Ansible job with options:")
     $log.log_hashes(opts, :filter => ["api_token", "token"])
-    new_job = ManageIQ::Providers::EmbeddedAnsible::AutomationManager::Job.create_job(jt, decrypt_options(opts))
+    new_job = ManageIQ::Providers::EmbeddedAnsible::AutomationManager::Job.create_job(my_playbook, decrypt_options(opts))
     update_job_for_playbook(action, new_job, opts[:hosts])
 
     _log.info("Ansible job with ref #{new_job.ems_ref} was created.")
@@ -109,30 +110,10 @@ def get_job_options(action)
     options[job_option_key(action)].deep_dup
   end
 
-  CONFIG_OPTIONS_WHITELIST = %i[
-    become_enabled
-    cloud_credential_id
-    credential_id
-    execution_ttl
-    extra_vars
-    hosts
-    network_credential_id
-    vault_credential_id
-    verbosity
-  ].freeze
-
   def config_options(action)
     options.fetch_path(:config_info, action.downcase.to_sym).slice(*CONFIG_OPTIONS_WHITELIST).with_indifferent_access
   end
 
-  def translate_credentials!(job_options)
-    %i[credential vault_credential network_credential cloud_credential].each do |cred|
-      cred_sym = "#{cred}_id".to_sym
-      credential_id = job_options.delete(cred_sym)
-      job_options[cred] = Authentication.find(credential_id).native_ref if credential_id.present?
-    end
-  end
-
   def save_job_options(action, overrides)
     job_options = config_options(action)
 
@@ -171,16 +152,6 @@ def extra_vars_from_dialog
     params.blank? ? {} : {:extra_vars => params}
   end
 
-  def use_default_inventory?(hosts)
-    hosts.blank? || hosts == 'localhost'
-  end
-
-  def hosts_array(hosts_string)
-    return ["localhost"] if use_default_inventory?(hosts_string)
-
-    hosts_string.split(',').map(&:strip).delete_blanks
-  end
-
   # update job attributes only available to playbook provisioning
   def update_job_for_playbook(action, job, hosts)
     playbook_id = options.fetch_path(:config_info, action.downcase.to_sym, :playbook_id)
@@ -195,12 +166,6 @@ def decrypt_options(opts)
 
   def log_stdout(action)
     log_option = options.fetch_path(:config_info, action.downcase.to_sym, :log_output) || 'on_error'
-    return unless %(on_error always).include?(log_option)
-    job = job(action)
-    return if log_option == 'on_error' && job.raw_status.succeeded?
-    _log.info("Stdout from ansible job #{job.name}: #{job.raw_stdout('txt_download')}")
-  rescue => err
-    _log.error("Failed to get stdout from ansible job #{job.name}")
-    _log.log_backtrace(err)
+    playbook_log_stdout(log_option, job(action))
   end
 end