Add OrchestrationTemplateRunner to queue up orchestration stack deployment

app/models/manageiq/providers/cloud_manager/orchestration_template_runner.rb

@@ -0,0 +1,165 @@
+class ManageIQ::Providers::CloudManager::OrchestrationTemplateRunner < ::Job
+  DEFAULT_EXECUTION_TTL = 10 # minutes
+
+  # options are job table columns, including options column which is the playbook context info
+  def self.create_job(options)
+    super(name, options)
+  end
+
+  def minimize_indirect
+    @minimize_indirect = true if @minimize_indirect.nil?
+    @minimize_indirect
+  end
+
+  def current_job_timeout(_timeout_adjustment = 1)
+    @execution_ttl ||=
+      (options[:execution_ttl].present? ? options[:execution_ttl].try(:to_i) : DEFAULT_EXECUTION_TTL) * 60
+  end
+
+  def start
+    time = Time.zone.now
+    update_attributes(:started_on => time)
+    miq_task.update_attributes(:started_on => time)
+    my_signal(false, :deploy_orchestration_stack, :priority => MiqQueue::HIGH_PRIORITY)
+  end
+
+  def update
+    time = Time.zone.now
+    update_attributes(:started_on => time)
+    miq_task.update_attributes(:started_on => time)
+    my_signal(false, :update_orchestration_stack)
+  end
+
+  def deploy_orchestration_stack
+    set_status('deploying orchestration stack')
+
+    @orchestration_stack = ManageIQ::Providers::CloudManager::OrchestrationStack.create_stack(
+      orchestration_manager, options[:stack_name], orchestration_template, options[:create_options]
+    )
+    options[:orchestration_stack_id] = @orchestration_stack.id
+    self.name = "#{name}, Orchestration Stack ID: #{@orchestration_stack.id}"
+    miq_task.update_attributes(:name => name)
+    save!
+    my_signal(false, :poll_stack_status, 10)
+  rescue StandardError => err
+    _log.log_backtrace(err)
+    my_signal(minimize_indirect, :post_stack_run, err.message, 'error')
+  end
+
+  def update_orchestration_stack
+    set_status('updating orchestration stack')
+
+    orchestration_stack.raw_update_stack(orchestration_template, options[:update_options])
+    self.name = "#{name}, update Orchestration Stack ID: #{orchestration_stack.id}"
+    miq_task.update_attributes(:name => name)
+    save!
+    my_signal(false, :poll_stack_status, 10)
+  rescue StandardError => err
+    _log.log_backtrace(err)
+    my_signal(minimize_indirect, :post_stack_run, err.message, 'error')
+  end
+
+  def poll_stack_status(interval)
+    set_status('checking orchestration stack deployment status')
+
+    status, message = orchestration_stack ? orchestration_stack.raw_status.normalized_status : ["check_status_failed", "stack has not been deployed"]
+    options.merge!(:orchestration_stack_status => status, :orchestration_stack_message => message)
+    save!
+    _log.info("Stack deployment status: #{status}, reason: #{message}")
+
+    case status.downcase
+    when 'create_complete', 'update_complete'
+      my_signal(minimize_indirect, :post_stack_run, "Orchestration stack [#{orchestration_stack.name}] #{status}", 'ok')
+    when 'rollback_complete', 'delete_complete', /failed$/, /canceled$/
+      _log.error("Orchestration stack deployment error: #{message}. Please examine stack resources for more details")
+      my_signal(minimize_indirect, :post_stack_run, "Orchestration stack deployment error: #{message}", 'error')
+    else
+      interval = 60 if interval > 60
+      my_signal(false, :poll_stack_status, interval * 2, :deliver_on => Time.now.utc + interval)
+    end
+  rescue MiqException::MiqOrchestrationStackNotExistError, MiqException::MiqOrchestrationStatusError => err
+    # naming convention requires status to end with "failed"
+    options.merge!(:orchestration_stack_status => 'check_status_failed', :orchestration_stack_message => err.message)
+    save!
+    _log.log_backtrace(err)
+    my_signal(minimize_indirect, :post_stack_run, err.message, 'error')
+  end
+
+  def post_stack_run(message, status)
+    my_signal(true, :finish, message, status)
+  end
+
+  def set_status(message, status = "ok")
+    _log.info(message)
+    super
+  end
+
+  def my_signal(no_queue, action, *args, deliver_on: nil, priority: MiqQueue::NORMAL_PRIORITY)
+    if no_queue
+      signal(action, *args)
+    else
+      queue_signal(action, *args, :deliver_on => deliver_on, :priority => priority)
+    end
+  end
+
+  def orchestration_stack
+    OrchestrationStack.find_by(:id => options[:orchestration_stack_id])
+  end
+
+  def orchestration_stack_status
+    options[:orchestration_stack_status]
+  end
+
+  def orchestration_stack_message
+    options[:orchestration_stack_message]
+  end
+
+  alias initializing dispatch_start
+  alias finish       process_finished
+  alias abort_job    process_abort
+  alias cancel       process_cancel
+  alias error        process_error
+
+  private
+
+  attr_writer :minimize_indirect
+
+  def load_transitions
+    self.state ||= 'initialize'
+
+    {
+      :initializing               => {'initialize'       => 'waiting_to_start'},
+      :start                      => {'waiting_to_start' => 'running'},
+      :update                     => {'waiting_to_start' => 'updating'},
+      :deploy_orchestration_stack => {'running'          => 'stack_job'},
+      :update_orchestration_stack => {'updating'         => 'stack_job'},
+      :poll_stack_status          => {'stack_job'        => 'stack_job'},
+      :post_stack_run             => {'stack_job'        => 'stack_done'},
+      :finish                     => {'*'                => 'finished'},
+      :abort_job                  => {'*'                => 'aborting'},
+      :cancel                     => {'*'                => 'canceling'},
+      :error                      => {'*'                => '*'}
+    }
+  end
+
+  def queue_signal(*args, deliver_on: nil, priority: MiqQueue::NORMAL_PRIORITY)
+    MiqQueue.put(
+      :class_name  => self.class.name,
+      :method_name => "signal",
+      :instance_id => id,
+      :priority    => priority,
+      :role        => 'ems_operations',
+      :zone        => options[:zone],
+      :args        => args,
+      :deliver_on  => deliver_on
+    )
+  end
+
+  def orchestration_manager
+    ExtManagementSystem.find_by(:id => options[:orchestration_manager_id])
+  end
+
+  def orchestration_template
+    OrchestrationTemplate.find_by(:id => options[:orchestration_template_id])
+  end
+end

app/models/service_orchestration.rb

@@ -17,27 +17,47 @@ def stack_name=(stname)
   end
 
   def orchestration_stack_status
-    return "check_status_failed", "stack has not been deployed" unless orchestration_stack
+    return ['deploy_failed', "can't find orchestration stack job for the service"] unless orchestration_runner_job
 
-    orchestration_stack.raw_status.normalized_status
-  rescue MiqException::MiqOrchestrationStackNotExistError, MiqException::MiqOrchestrationStatusError => err
-    # naming convention requires status to end with "failed"
-    ["check_status_failed", err.message]
+    return ['deploy_failed', orchestration_runner_job.message] if orchestration_runner_job.status == 'error'
+
+    return ['deploy_active', 'waiting for the orchestration stack status'] unless orchestration_runner_job.orchestration_stack_status
+
+    [orchestration_runner_job.orchestration_stack_status, orchestration_runner_job.orchestration_stack_message]
   end
 
   def deploy_orchestration_stack
-    creation_options = stack_options
-    @orchestration_stack = ManageIQ::Providers::CloudManager::OrchestrationStack.create_stack(
-      orchestration_manager, stack_name, orchestration_template, creation_options
-    )
+    deploy_stack_options = stack_options
+    job_options = {
+      :create_options            => deploy_stack_options,
+      :orchestration_manager_id  => orchestration_manager.id,
+      :orchestration_template_id => orchestration_template.id,
+      :stack_name                => stack_name,
+      :zone                      => my_zone
+    }
+
+    @deploy_stack_job = ManageIQ::Providers::CloudManager::OrchestrationTemplateRunner.create_job(job_options)
+    update_attributes(:options => options.merge(:deploy_stack_job_id => @deploy_stack_job.id))
+    @deploy_stack_job.signal(:start)
+
+    wait_on_orchestration_stack
+    orchestration_stack
   ensure
     # create options may never be saved before unless they were overridden
     save_create_options
   end
 
   def update_orchestration_stack
-    # use orchestration_template from service_template, which may be different from existing orchestration_template
-    orchestration_stack.raw_update_stack(service_template.orchestration_template, update_options)
+    job_options = {
+      # use orchestration_template from service_template, which may be different from existing orchestration_template
+      :orchestration_template_id => service_template.orchestration_template.id,
+      :orchestration_stack_id    => orchestration_stack.id,
+      :update_options            => update_options,
+      :zone                      => my_zone
+    }
+    @update_stack_job = ManageIQ::Providers::CloudManager::OrchestrationTemplateRunner.create_job(job_options)
+    update_attributes(:options => options.merge(:update_stack_job_id => @update_stack_job.id))
+    @update_stack_job.signal(:update)
   end
 
   def orchestration_stack
@@ -47,6 +67,7 @@ def orchestration_stack
     if @orchestration_stack.nil? && options.fetch_path(:orchestration_stack, 'ems_id')
       @orchestration_stack = OrchestrationStack.new(options[:orchestration_stack])
     end
+
     @orchestration_stack
   end
 
@@ -107,8 +128,8 @@ def add_stack_to_resource
 
   def link_orchestration_template
     # some orchestration stacks do not have associations with their templates in their provider, we can link them here
-    return if @orchestration_stack.nil? || @orchestration_stack.orchestration_template
-    @orchestration_stack.update_attributes(:orchestration_template => orchestration_template)
+    return if orchestration_stack.nil? || orchestration_stack.orchestration_template
+    orchestration_stack.update_attributes(:orchestration_template => orchestration_template)
   end
 
   def assign_vms_owner
@@ -146,8 +167,8 @@ def pick_orchestration_template(dialog_options)
   end
 
   def save_create_options
-    stack_attributes = if @orchestration_stack
-                         @orchestration_stack.attributes.compact
+    stack_attributes = if orchestration_stack
+                         orchestration_stack.attributes.compact
                        else
                          {:name => stack_name}
                        end
@@ -156,4 +177,28 @@ def save_create_options
                    :create_options      => dup_and_process_password(stack_options))
     save!
   end
+
+  def deploy_stack_job
+    @deploy_stack_job ||= Job.find_by(:id => options.fetch_path(:deploy_stack_job_id))
+  end
+
+  def update_stack_job
+    @update_stack_job ||= Job.find_by(:id => options.fetch_path(:update_stack_job_id))
+  end
+
+  def orchestration_runner_job
+    update_stack_job || deploy_stack_job
+  end
+
+  def wait_on_orchestration_stack
+    while deploy_stack_job.orchestration_stack.blank?
+      _log.info("Waiting for the deployment of orchestration stack [#{stack_name}]...")
+      sleep 2
+      # Code running with Rails QueryCache enabled,
+      # need to disable caching for the reload to see updates.
+      self.class.uncached { reload }
+      deploy_stack_job.class.uncached { deploy_stack_job.reload }
+    end
+    @orchestration_stack = deploy_stack_job.orchestration_stack
+  end
 end