feature(bosh-pipeline): add manual-errand support and multi job errands

We also use metadata-resource to replace secrets and paas-template resources,
so we are able to trigger errands even on git reset.

close #14, close #285, related to #197

concourse/pipelines/template/bosh-pipeline.yml.erb

@@ -165,7 +165,7 @@ resources:
  deployment: <%= name %>
  ca_cert: <%= bosh_cert[depls]&.dump %>
 
- <% if deployment_details.errands? %>
+ <% if deployment_details.errands? || deployment_details.manual_errands? %>
 - name: errand-<%= name %>
  type: bosh-errand
  source:
@@ -666,7 +666,6 @@ jobs:
  - ops-and-vars-files/vars/*
  cleanup: true
 
-
  - task: bosh-variables
  input_mapping: {scripts-resource: cf-ops-automation, secrets: secrets-full-writer}
  output_mapping: {result-dir: bosh-variables-result}
@@ -711,9 +710,10 @@ jobs:
  SECRETS_DIR: credentials-resource/<%= depls %>/<%= name %>
 
  <% if deployment_details.errands? %>
-- name: run-errand-<%= name %>
- <% jobs["deploy-#{name[0]}*"] << "run-errand-#{name}" %>
- serial: true
+ <% deployment_details.errands.each do |errand_name, errand_info| %>
+- name: run-errand-<%= name %>-<%= errand_name %>
+ <% jobs["deploy-#{name[0]}*"] << "run-errand-#{name}-#{errand_name}" %>
+ serial_groups: [auto-errand-<%= name %>]
  on_failure:
  put: failure-alert
  params:
@@ -723,20 +723,37 @@ jobs:
  username: Concourse
  plan:
  - in_parallel:
- - get: <%= deployment_details.select_secrets_scan_repository("secrets-#{name}", "secrets-full-writer") %>
- params: { submodules: none}
- trigger: <%= deployment_details.local_deployment_secrets_trigger? %>
- passed: [deploy-<%= name %>]
- - get: paas-templates-<%= name %>
- params: { submodules: none}
+ - get: concourse-meta
  trigger: true
  passed: [deploy-<%= name %>]
- <% deployment_details.errands.each do |errand_name, errand_info| %>
+
  - put: errand-<%= name %>
  params:
  name: <%= errand_name %>
  <% end %>
  <% end %>
+
+ <% if deployment_details.manual_errands? %>
+ <% deployment_details.manual_errands.each do |errand_name, errand_info| %>
+- name: run-manual-errand-<%= name %>-<%= errand_name %>
+ <% jobs["deploy-#{name[0]}*"] << "run-manual-errand-#{name}-#{errand_name}" %>
+ serial: true
+ on_failure:
+ put: failure-alert
+ params:
+ channel: ((slack-channel))
+ text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
+ icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
+ username: Concourse
+ plan:
+ - in_parallel:
+ - get: concourse-meta
+ passed: [deploy-<%= name %>]
+ - put: errand-<%= name %>
+ params:
+ name: <%= errand_name %>
+ <% end %>
+ <% end %>
 <% end %>
 
 <% if enabled_deployments.any? %>

docs/reference_dataset/template_repository/hello-world-root-depls/bosh-deployment-sample/deployment-dependencies.yml

@@ -23,3 +23,9 @@ deployment:
  vault:
  base_location: http://github.com/ # COA can use boshrelease published on github if a tgz is available
  repository: cloudfoundry-community/vault-boshrelease
+ errands: # errands to execute automatically after each deploy. Errand are executed one by one in random order
+ # errand-1:
+ # errand-2:
+ manual-errands: # errands manually executed by an operator
+ # manual-errand-1:
+ # manual-errand-2:

lib/pipeline_helpers/deployment_details.rb

@@ -46,6 +46,14 @@ def errands
  @details['errands'] || DEFAULT_EMPTY_ERRANDS_VALUE
  end
 
+ def manual_errands?
+ @details['manual-errands']&.any?
+ end
+
+ def manual_errands
+ @details['manual-errands'] || DEFAULT_EMPTY_ERRANDS_VALUE
+ end
+
  def releases
  @details['releases']&.sort || DEFAULT_EMPTY_RELEASES_VALUE
  end

scripts/concourse-manual-pipelines-update.rb

@@ -7,6 +7,7 @@
 # Argument parsing
 OPTIONS = {
  depls: 'ops-depls',
+ team: 'main',
  no_interactive: false,
  fail_fast: false,
  fail_on_error: true,
@@ -53,6 +54,9 @@
  opts.on('--[no-]use-coa-config', "Use configuration - Default: #{OPTIONS[:coa_config]}") do |boolean_option|
  OPTIONS[:coa_config] = boolean_option
  end
+ opts.on('--team=TEAM', "Concourse team to use to deploy pipelines - Default: #{OPTIONS[:team]}") do |team_string|
+ OPTIONS[:team] = team_string
+ end
 end
 opt_parser.parse!
 
@@ -72,11 +76,17 @@ def get_pipeline_name(name)
  "#{PIPELINE_PREFIX}#{name}"
 end
 
-def set_pipeline(target_name:, name:, config:, load: [], options: [])
+def set_pipeline(target_name:, team_name: 'main', name:, config:, load: [], options: [])
  return if OPTIONS.key?(:match) && !name.include?(OPTIONS[:match])
  return if OPTIONS.key?(:without) && name.include?(OPTIONS[:without])
+
  puts " Setting #{name} pipeline"
 
+ switch_team_cmd = %{bash -c "fly -t #{target_name} edit-target -n #{team_name}"}
+ switch_concourse_team = system(switch_team_cmd)
+ puts "Switched to team: #{team_name}"
+ raise "Failed to switch team to #{team_name} to load pipeline #{get_pipeline_name(name)} from template #{name}" unless switch_concourse_team
+
  fly_cmd = %{bash -c "fly -t #{target_name} set-pipeline \
  -p #{get_pipeline_name(name)} \
  -c #{config} \
@@ -88,9 +98,8 @@ def set_pipeline(target_name:, name:, config:, load: [], options: [])
 
  pipeline_successfully_loaded = system(fly_cmd)
  puts "Pipeline successfully loaded: #{pipeline_successfully_loaded}"
- if OPTIONS[:fail_fast] && !pipeline_successfully_loaded
- raise "Failed to load pipeline #{get_pipeline_name(name)} from template #{name}"
- end
+ raise "Failed to load pipeline #{get_pipeline_name(name)} from template #{name}" if OPTIONS[:fail_fast] && !pipeline_successfully_loaded
+
  pipeline_successfully_loaded
 end
 
@@ -126,9 +135,12 @@ def concourse_additional_options
 
 def load_pipeline_into_concourse(pipeline_name, pipeline_vars_files, pipeline_definition_filename, concourse_target_name)
  raise "No vars_files detected. Please ensure coa-config option is #{OPTIONS[:coa_config]}" if pipeline_vars_files&.empty?
+ pipeline_team_name = OPTIONS[:team]
+
  set_pipeline(
  target_name: concourse_target_name,
  name: pipeline_name,
+ team_name: pipeline_team_name,
  config: pipeline_definition_filename,
  load: pipeline_vars_files,
  options: concourse_additional_options
@@ -184,9 +196,7 @@ def update_pipelines(target_name)
  loaded_pipelines_status[pipeline_name] = pipeline_loading_status
  end
 
- if OPTIONS[:fail_on_error]
- raise "pipeline loading error. Summary #{loaded_pipelines_status}" unless loaded_pipelines_status.select { |_, status| !status.nil? && !status }.empty?
- end
+ raise "pipeline loading error. Summary #{loaded_pipelines_status}" unless loaded_pipelines_status.select { |_, status| !status.nil? && !status }.empty? if OPTIONS[:fail_on_error]
 end
 
 update_pipelines TARGET_NAME