add lockfile bump and release notes draft tools

Fixes #10265

Author: jsvd

rakelib/bump_plugin_versions.rake

@@ -0,0 +1,99 @@
+require 'net/http'
+require 'uri'
+require 'fileutils'
+require 'yaml'
+
+def compute_dependecy(version, allow_for)
+  gem_version = Gem::Version.new(version)
+  return version if gem_version.prerelease?
+  major, minor, patch = gem_version.release.segments
+  case allow_for
+  when "major"
+    then "~> #{major}"
+  when "minor"
+    then "~> #{major}.#{minor}"
+  when "patch"
+    then "~> #{major}.#{minor}.#{patch}"
+  end
+end
+
+base_branch = ARGV[0]
+base_logstash_version = ARGV[1]
+allow_bump_for = ARGV[2]
+
+unless ["major", "minor", "patch"].include?(allow_bump_for)
+  puts "second argument must be one of 'major', 'minor' or 'patch', got '#{allow_bump_for}'"
+  exit(1)
+end
+
+puts "Computing #{allow_bump_for} plugin dependency bump from #{base_logstash_version}.."
+
+puts "Fetching lock file for #{base_logstash_version}.."
+uri = URI.parse("https://raw.githubusercontent.com/elastic/logstash/v#{base_logstash_version}/Gemfile.jruby-2.3.lock.release")
+result = Net::HTTP.get(uri)
+if result.match(/404/)
+  puts "Lock file or git tag for #{base_logstash_version} not found. Aborting"
+  exit(1)
+end
+
+base_plugin_versions = {}
+skip_elements = ["logstash-core", "logstash-devutils", "logstash-core-plugin-api"]
+result.split("\n").each do |line|
+  # match e.g. "    logstash-output-nagios (3.0.6)"
+  if match = line.match(/^    (?<plugin>logstash-.+?)\s\((?<version>.+?)(?:-java)?\)/)
+    next if skip_elements.include?(match["plugin"])
+    base_plugin_versions[match["plugin"]] = match["version"]
+  end
+end
+
+computed_dependency = {}
+puts "Generating new Gemfile.template file with computed dependencies"
+gemfile = IO.read("Gemfile.template")
+base_plugin_versions.each do |plugin, version|
+  dependency = compute_dependecy(version, allow_bump_for)
+  gemfile.gsub!(/"#{plugin}".*$/, "\"#{plugin}\", \"#{dependency}\"")
+end
+
+IO.write("Gemfile.template", gemfile)
+
+puts "Cleaning up before running 'rake artifact:tar'"
+FileUtils.rm_f("Gemfile")
+FileUtils.rm_f("Gemfile.jruby-2.3.lock.release")
+FileUtils.rm_rf("vendor")
+
+# compute new lock file
+puts "Running 'rake artifact:tar'"
+result = `rake artifact:tar`
+
+puts "Cleaning up generated lock file (removing injected requirements)"
+# remove explicit requirements from lock file
+lock_file = IO.read("Gemfile.lock")
+new_lock = []
+lock_file.split("\n").each do |line|
+  new_lock << line.gsub(/^  (?<plugin>logstash-\w+-.+?) .+?$/, "  \\k<plugin>")
+end
+IO.write("Gemfile.lock", new_lock.join("\n"))
+
+# rename file
+puts "Finishing up.."
+FileUtils.mv("Gemfile.lock", "Gemfile.jruby-2.3.lock.release")
+
+`git checkout -- Gemfile.template`
+
+puts `git diff Gemfile.jruby-2.3.lock.release`
+
+puts "Creating commit.."
+
+branch_name = "update_lock_#{Time.now.to_i}"
+`git checkout -b #{branch_name}`
+`git commit Gemfile.jruby-2.3.lock.release -m "Update #{allow_bump_for} plugin versions in gemfile lock"`
+
+puts "Pushing commit.."
+`git remote add upstream git@github.com:elastic/logstash.git`
+`git push upstream #{branch_name}`
+
+current_release = YAML.parse(IO.read("versions.yml"))["logstash"]
+puts "Creating Pull Request"
+pr_title = "bump lock file for #{current_release}"
+
+`curl -H "Authorization: token #{ENV['GITHUB_TOKEN']}" -d '{"title":"#{pr_title}","base":"#{base_branch}", "head":"#{branch_name}"}' https://api.github.com/repos/elastic/logstash/pulls`

tools/release/bump_plugin_versions.rb

@@ -0,0 +1,107 @@
+# Example:
+# ruby generate_release_notes.rb 6.4 6.4.1
+# 
+# This:
+# * compares the lock file of two commits
+# * for each plugin version bumped show CHANGELOG.md of the bumped version
+require 'tempfile'
+require 'yaml'
+require 'json'
+require 'net/http'
+
+RELEASE_NOTES_PATH = "docs/static/releasenotes.asciidoc"
+release_branch = ARGV[0]
+previous_release_tag = ARGV[1]
+report = []
+
+`git checkout #{release_branch}`
+
+current_release = YAML.load(IO.read("versions.yml"))["logstash"]
+current_release_dashes = current_release.tr(".", "-")
+
+release_notes = IO.read(RELEASE_NOTES_PATH).split("\n")
+
+release_notes.insert(5, "* <<logstash-#{current_release_dashes},Logstash #{current_release}>>")
+
+release_notes_entry_index = release_notes.find_index {|line| line.match(/^\[\[logstash/) }
+
+report << "[[logstash-#{current_release_dashes}]]"
+report << "=== Logstash #{current_release} Release Notes\n"
+
+plugin_changes = {}
+
+report <<  "---------- DELETE FROM HERE ------------"
+report <<  "=== Logstash Pull Requests with label v#{current_release}\n"
+
+uri = URI.parse("https://api.github.com/search/issues?q=repo:elastic/logstash+is:pr+is:closed+label:v#{current_release}&sort=created&order=asc")
+pull_requests = JSON.parse(Net::HTTP.get(uri))
+pull_requests['items'].each do |prs|
+  report << "* #{prs['title']} #{prs['html_url']}[##{prs['number']}]"
+end
+report << ""
+
+report <<  "=== Logstash Commits between #{release_branch} and #{previous_release_tag}\n"
+report <<  "Computed with \"git log --pretty=format:'%h -%d %s (%cr) <%an>' --abbrev-commit --date=relative v#{previous_release_tag}..#{release_branch}\""
+report <<  ""
+logstash_prs =  `git log --pretty=format:'%h -%d %s (%cr) <%an>' --abbrev-commit --date=relative v#{previous_release_tag}..#{release_branch}`
+report <<  logstash_prs
+report << "\n=== Logstash Plugin Release Changelogs ==="
+report << "Computed from \"git diff v#{previous_release_tag}..#{release_branch} *.release\""
+result = `git diff v#{previous_release_tag}..#{release_branch} *.release`.split("\n")
+
+result.each do |line|
+  # example "+    logstash-input-syslog (3.4.1)"
+  if match = line.match(/\+\s+(?<plugin>logstash-.+?-.+?)\s+\((?<version>\d+\.\d+.\d+).*?\)/)
+    plugin_changes[match[:plugin]] ||= []
+    plugin_changes[match[:plugin]] << match[:version]
+  elsif match = line.match(/\-\s+(?<plugin>logstash-.+?-.+?)\s+\((?<version>\d+\.\d+.\d+).*?\)/)
+    plugin_changes[match[:plugin]] ||= []
+    plugin_changes[match[:plugin]].unshift(match[:version])
+  else
+    # ..
+  end
+end
+report << "Changed plugin versions:"
+plugin_changes.each {|p, v| report << "#{p}: #{v.first} -> #{v.last}" }
+report << "---------- DELETE UP TO HERE ------------\n"
+
+report << "==== Plugins\n"
+
+plugin_changes.each do |plugin, versions|
+  _, type, name = plugin.split("-")
+  header = "*#{name.capitalize} #{type.capitalize}*"
+  start_changelog_file = Tempfile.new(plugin + 'start')
+  end_changelog_file = Tempfile.new(plugin + 'end')
+  changelog = `curl https://raw.githubusercontent.com/logstash-plugins/#{plugin}/v#{versions.last}/CHANGELOG.md`.split("\n")
+  report << "#{header}\n"
+  changelog.each do |line|
+    break if line.match(/^## #{versions.first}/)
+    next if line.match(/^##/)
+    line.gsub!(/^\+/, "")
+    line.gsub!(/ #(?<number>\d+)\s*$/, " https://github.com/logstash-plugins/#{plugin}/issues/\\k<number>[#\\k<number>]")
+    line.gsub!(/^\s+-/, "*")
+    report << line
+  end
+  report << ""
+  start_changelog_file.unlink
+  end_changelog_file.unlink
+end
+
+release_notes.insert(release_notes_entry_index, report.join("\n").gsub(/\n{3,}/, "\n\n"))
+
+IO.write(RELEASE_NOTES_PATH, release_notes.join("\n"))
+
+puts "Creating commit.."
+branch_name = "update_release_notes_#{Time.now.to_i}"
+`git checkout -b #{branch_name}`
+`git commit docs/static/releasenotes.asciidoc -m "Update release notes for #{current_release}"`
+
+puts "Pushing commit.."
+`git remote add upstream git@github.com:elastic/logstash.git`
+`git push upstream #{branch_name}`
+
+puts "Creating Pull Request"
+pr_title = "Release notes draft for #{current_release}"
+`curl -H "Authorization: token #{ENV['GITHUB_TOKEN']}" -d '{"title":"#{pr_title}","base":"#{ENV['branch_specifier']}", "head":"#{branch_name}"}' https://api.github.com/repos/elastic/logstash/pulls`
+
+puts "Done"