From bae314678f30ba771edfd56056fc813c61aa375c Mon Sep 17 00:00:00 2001 From: Jacob Morgan Date: Fri, 30 Oct 2020 14:56:56 +1300 Subject: [PATCH] Custom dependabot script for locally hosted gitlab server * Custom build of the dependabot core for https://github.com/dependabot/dependabot-core/pull/1848 * Group packages by package name or all * Custom message builder --- Gemfile | 5 +- Gemfile.lock | 140 ++++++------ dependabot-config.rb | 95 ++++++++ gitlab-processor.rb | 531 +++++++++++++++++++++++++++++++++++++++++++ message_builder.rb | 94 ++++++++ run.rb | 35 +++ 6 files changed, 833 insertions(+), 67 deletions(-) create mode 100644 dependabot-config.rb create mode 100644 gitlab-processor.rb create mode 100644 message_builder.rb create mode 100644 run.rb diff --git a/Gemfile b/Gemfile index 798f0f38..987894b0 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,7 @@ # frozen_string_literal: true -source "https://rubygems.org" +ruby "2.6.6" +source "http://nuget.pus.local/rubygems/rubygems" +gem "dependabot-omnibus", "~> 0.123.1.pharos.1" gem "irb" -gem "dependabot-omnibus", "~> 0.118.8" diff --git a/Gemfile.lock b/Gemfile.lock index 73dd4490..c901c50c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,33 +1,35 @@ GEM - remote: https://rubygems.org/ + remote: http://nuget.pus.local/rubygems/rubygems/ specs: addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) ast (2.4.1) aws-eventstream (1.1.0) - aws-partitions (1.349.0) - aws-sdk-codecommit (1.37.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-partitions (1.383.0) + aws-sdk-codecommit (1.40.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.104.3) + aws-sdk-core (3.109.1) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-ecr (1.35.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-ecr (1.39.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sigv4 (1.2.1) + aws-sigv4 (1.2.2) aws-eventstream (~> 1, >= 1.0.2) citrus (3.0.2) commonmarker (0.21.0) ruby-enum (~> 0.5) - concurrent-ruby (1.1.6) - dependabot-bundler (0.118.8) - dependabot-common (= 0.118.8) - dependabot-cargo (0.118.8) - dependabot-common (= 0.118.8) - dependabot-common (0.118.8) + concurrent-ruby (1.1.7) + dependabot-bundler (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-cake (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-cargo (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-common (0.123.1.pharos.1) aws-sdk-codecommit (~> 1.28) aws-sdk-ecr (~> 1.5) bundler (>= 1.16, < 3.0.0) @@ -35,65 +37,68 @@ GEM docker_registry2 (~> 1.7, >= 1.7.1) excon (~> 0.75) gitlab (= 4.16.1) + inifile (~> 3.0) nokogiri (~> 1.8) octokit (~> 4.6) pandoc-ruby (~> 2.0) parseconfig (~> 1.0) parser (~> 2.5) toml-rb (>= 1.1.2, < 3.0) - dependabot-composer (0.118.8) - dependabot-common (= 0.118.8) - dependabot-dep (0.118.8) - dependabot-common (= 0.118.8) - dependabot-docker (0.118.8) - dependabot-common (= 0.118.8) - dependabot-elm (0.118.8) - dependabot-common (= 0.118.8) - dependabot-git_submodules (0.118.8) - dependabot-common (= 0.118.8) - dependabot-github_actions (0.118.8) - dependabot-common (= 0.118.8) - dependabot-go_modules (0.118.8) - dependabot-common (= 0.118.8) - dependabot-gradle (0.118.8) - dependabot-common (= 0.118.8) - dependabot-hex (0.118.8) - dependabot-common (= 0.118.8) - dependabot-maven (0.118.8) - dependabot-common (= 0.118.8) - dependabot-npm_and_yarn (0.118.8) - dependabot-common (= 0.118.8) - dependabot-nuget (0.118.8) - dependabot-common (= 0.118.8) - dependabot-omnibus (0.118.8) - dependabot-bundler (= 0.118.8) - dependabot-cargo (= 0.118.8) - dependabot-common (= 0.118.8) - dependabot-composer (= 0.118.8) - dependabot-dep (= 0.118.8) - dependabot-docker (= 0.118.8) - dependabot-elm (= 0.118.8) - dependabot-git_submodules (= 0.118.8) - dependabot-github_actions (= 0.118.8) - dependabot-go_modules (= 0.118.8) - dependabot-gradle (= 0.118.8) - dependabot-hex (= 0.118.8) - dependabot-maven (= 0.118.8) - dependabot-npm_and_yarn (= 0.118.8) - dependabot-nuget (= 0.118.8) - dependabot-python (= 0.118.8) - dependabot-terraform (= 0.118.8) - dependabot-python (0.118.8) - dependabot-common (= 0.118.8) - dependabot-terraform (0.118.8) - dependabot-common (= 0.118.8) + dependabot-composer (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-dep (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-docker (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-elm (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-git_submodules (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-github_actions (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-go_modules (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-gradle (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-hex (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-maven (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-npm_and_yarn (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-nuget (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-omnibus (0.123.1.pharos.1) + dependabot-bundler (= 0.123.1.pharos.1) + dependabot-cake (= 0.123.1.pharos.1) + dependabot-cargo (= 0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-composer (= 0.123.1.pharos.1) + dependabot-dep (= 0.123.1.pharos.1) + dependabot-docker (= 0.123.1.pharos.1) + dependabot-elm (= 0.123.1.pharos.1) + dependabot-git_submodules (= 0.123.1.pharos.1) + dependabot-github_actions (= 0.123.1.pharos.1) + dependabot-go_modules (= 0.123.1.pharos.1) + dependabot-gradle (= 0.123.1.pharos.1) + dependabot-hex (= 0.123.1.pharos.1) + dependabot-maven (= 0.123.1.pharos.1) + dependabot-npm_and_yarn (= 0.123.1.pharos.1) + dependabot-nuget (= 0.123.1.pharos.1) + dependabot-python (= 0.123.1.pharos.1) + dependabot-terraform (= 0.123.1.pharos.1) + dependabot-python (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) + dependabot-terraform (0.123.1.pharos.1) + dependabot-common (= 0.123.1.pharos.1) docker_registry2 (1.9.0) rest-client (>= 1.8.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - excon (0.76.0) - faraday (1.0.1) + excon (0.78.0) + faraday (1.1.0) multipart-post (>= 1.2, < 3) + ruby2_keywords gitlab (4.16.1) httparty (~> 0.14, >= 0.14.0) terminal-table (~> 1.5, >= 1.5.1) @@ -105,6 +110,7 @@ GEM multi_xml (>= 0.5.2) i18n (1.8.5) concurrent-ruby (~> 1.0) + inifile (3.0.0) irb (1.2.0) reline (>= 0.0.1) jmespath (1.4.0) @@ -122,9 +128,9 @@ GEM sawyer (~> 0.8.0, >= 0.5.3) pandoc-ruby (2.1.4) parseconfig (1.0.8) - parser (2.7.1.4) + parser (2.7.2.0) ast (~> 2.4.1) - public_suffix (4.0.5) + public_suffix (4.0.6) reline (0.0.7) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) @@ -133,6 +139,7 @@ GEM netrc (~> 0.8) ruby-enum (0.8.0) i18n + ruby2_keywords (0.0.2) sawyer (0.8.2) addressable (>= 2.3.5) faraday (> 0.8, < 2.0) @@ -149,8 +156,11 @@ PLATFORMS ruby DEPENDENCIES - dependabot-omnibus (~> 0.118.8) + dependabot-omnibus (~> 0.123.1.pharos.1) irb +RUBY VERSION + ruby 2.6.6p146 + BUNDLED WITH 1.17.3 diff --git a/dependabot-config.rb b/dependabot-config.rb new file mode 100644 index 00000000..888d864e --- /dev/null +++ b/dependabot-config.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +class DependabotConfig + def initialize(config) + @package_manager = config["package_manager"] + @directory = config["directory"] + @update_schedule = config["update_schedule"] + @target_branch = config["target_branch"] + + @ignored_updates = [] + unless config["ignored_updates"].nil? + @ignored_updates = config["ignored_updates"].map do |ignored_update| + IgnoredUpdate.new(ignored_update["match"]) + end + end + + @automerged_updates = [] + unless config["automerged_updates"].nil? + @automerged_updates = config["automerged_updates"].map do |automerged_update| + AutomergedUpdate.new(automerged_update["match"]) + end + end + + @group_updates = [GroupUpdate.new({ "dependency_name" => "*" })] + unless config["group_updates"].nil? + @group_updates = config["group_updates"].map do |group_update| + GroupUpdate.new(group_update["match"]) + end + end + + @commit_message = {} + unless config["commit_message"].nil? + @commit_message = CommitMessage.new(config["commit_message"]).to_h + end + end + + attr_reader :package_manager + attr_reader :directory + attr_reader :update_schedule + attr_reader :target_branch + attr_reader :ignored_updates + attr_reader :automerged_updates + attr_reader :group_updates + attr_reader :commit_message + + class IgnoredUpdate + def initialize(config) + @dependency_name = config["dependency_name"] + @version_requirement = config["version_requirement"] + end + + attr_reader :dependency_name + attr_reader :version_requirement + end + + class AutomergedUpdate + def initialize(config) + @dependency_name = config["dependency_name"] + @dependency_type = config["dependency_type"] + @update_type = config["update_type"] + end + + attr_reader :dependency_name + attr_reader :dependency_type + attr_reader :update_type + end + + class GroupUpdate + def initialize(config) + @dependency_name = config["dependency_name"] + end + + attr_reader :dependency_name + end + + class CommitMessage + def initialize(config) + @prefix = config["prefix"] + @prefix_development = config["prefix_development"] + @include_scope = config["include_scope"] + end + + attr_reader :prefix + attr_reader :prefix_development + attr_reader :include_scope + + def to_h + { + prefix: @prefix, + prefix_development: @prefix_development, + include_scope: @include_scope + } + end + end +end diff --git a/gitlab-processor.rb b/gitlab-processor.rb new file mode 100644 index 00000000..af575a7e --- /dev/null +++ b/gitlab-processor.rb @@ -0,0 +1,531 @@ +# frozen_string_literal: true + +require "dependabot/omnibus" +require "dependabot/hex/version" +require_relative "dependabot-config" +require_relative "message_builder" + +# MonkeyPatch: dependabot to avoid issues relating to https://github.com/dependabot/dependabot-core/pull/1472 +Dependabot::Nuget::UpdateChecker::RepositoryFinder.class_eval do + def build_v2_url(response, repo_details) + doc = Nokogiri::XML(response.body) + doc.remove_namespaces! + base_url = doc.at_xpath("service")&.attributes&.fetch("base", nil)&.value + + base_url ||= repo_details.fetch(:url) + return unless base_url + + { + repository_url: base_url, + versions_url: File.join( + base_url, + "FindPackagesById()?id='#{dependency.name}'" + ), + auth_header: auth_header_for_token(repo_details.fetch(:token)), + repository_type: "v2" + } + end +end + +Dependabot::Source.class_eval do + PHAROS_SOURCE = %r{ + (?git-us.pharos) + (?:\.com)[/:] + (?[\w.-]+/(?:(?!\.git|\.\s)[\w.-])+) + (?:(?:/tree|/blob)/(?[^/]+)/(?.*)[\#|/])? + }x.freeze + + CUSTOM_SOURCE_REGEX = / + (?:#{Dependabot::Source::SOURCE_REGEX})| + (?:#{PHAROS_SOURCE}) + /x.freeze + + def self.from_url(url_string) + return unless url_string&.match?(CUSTOM_SOURCE_REGEX) + + captures = url_string.match(CUSTOM_SOURCE_REGEX).named_captures + + if captures.fetch("provider") == "git-us.pharos" + return new( + provider: "gitlab", + repo: captures.fetch("repo"), + directory: captures.fetch("directory"), + branch: captures.fetch("branch"), + hostname: "git-us.pharos.com", + api_endpoint: "https://git-us.pharos.com/api/v4" + ) + end + + new( + provider: captures.fetch("provider"), + repo: captures.fetch("repo"), + directory: captures.fetch("directory"), + branch: captures.fetch("branch") + ) + end +end + +Dependabot::PullRequestCreator::PrNamePrefixer.class_eval do + def capitalize_first_word? + if commit_message_options.key?(:prefix) + return true if Dependabot::PullRequestCreator::PrNamePrefixer::ANGULAR_PREFIXES.include?(commit_message_options[:prefix]) + + return !commit_message_options[:prefix]&.strip&.match?(/\A[a-z]/) + end + + return capitalise_first_word_from_last_dependabot_commit_style if last_dependabot_commit_style + + capitalise_first_word_from_previous_commits + end +end + +Dependabot::MetadataFinders::Base::CommitsFinder.class_eval do + def gitlab_client + @gitlab_client ||= if source.hostname == "git-us.pharos.com" + Dependabot::Clients::GitlabWithRetries.for_source(source: source, credentials: credentials) + else + Dependabot::Clients::GitlabWithRetries.for_gitlab_dot_com(credentials: credentials) + end + end +end + +# noinspection RubyResolve +class GitLabProcessor + def initialize(organisation, available_credentials) + @organisation = organisation + @available_credentials = available_credentials + + @organisation_credentials = available_credentials. + select { |cred| cred["type"] == "git_source" }. + find { |cred| cred["host"] == "git-us.pharos.com" } + + @gitlab_client = Gitlab.client( + endpoint: "https://#{@organisation_credentials&.fetch('host')}/api/v4", + private_token: @organisation_credentials&.fetch("password") + ) + + @merge_request_author = "dependabot" + end + + def process(repository_name, mr_limit_per_repo) + puts "#{@organisation} => #{repository_name} => Checking for repository...." + project = @gitlab_client.project(repository_name) + process_project(project, mr_limit_per_repo) + rescue Gitlab::Error::NotFound + puts "#{@organisation} => #{repository_name} => Named repository not found." + rescue Gitlab::Error::Forbidden + puts "#{@organisation} => #{repository_name} => Access not granted to repository" + end + + def process_project(project, mr_limit_per_repo) + puts "#{@organisation} => #{project.path_with_namespace} => Checking for Depenadbot configuration file..." + response_file = @gitlab_client.get_file(project.id, ".dependabot/config.yml", "master") + response_config = Base64.decode64(response_file.content) + config = YAML.safe_load(response_config) + config["update_configs"].each do |update_config| + process_dependabot_config(project, DependabotConfig.new(update_config), mr_limit_per_repo) + end + rescue Gitlab::Error::NotFound => e + generate_bug_dependabot_config(project, e) + rescue Gitlab::Error::Forbidden => e + generate_bug_dependabot_config(project, e) + end + + def generate_bug_dependabot_config(project, error) + puts "#{@organisation} => #{project.path_with_namespace} => Dependabot configuration file issue, raising bug if required..." + puts error.message + end + + def process_dependabot_config(project, dependabot_config, mr_limit_per_repo) + # not supported: target_branch, default_reviewers, default_assignees, default_labels, allowed_updates, version_requirement_updates + # not supported: ignored_updates.version_requirement + # not supported: automerged_updates.dependency_type, limited support for automerged_updates.update_type + + package_manager = package_manager(dependabot_config) + + update_schedule = case dependabot_config.update_schedule + when "live" then true + when "daily" then true + when "weekly" then Date.today.strftime("%w").to_i == 1 + when "monthly" then Date.today.strftime("%e").to_i == 1 + else raise "Unsupported update schedule: #{dependabot_config.update_schedule}" + end + if update_schedule == false + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => Skipping dependency checking" + return + end + + open_merge_requests = @gitlab_client.merge_requests(project.id, { state: "opened" }) + if open_merge_requests.count { |merge_request| merge_request.author.username == @merge_request_author } == mr_limit_per_repo + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => Skipping; maximum number of allowed opened MR requests has been reached" + return + end + + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => Dependabot configuration { directory: #{dependabot_config.directory} }" + source = Dependabot::Source.new( + provider: "gitlab", + hostname: @organisation_credentials&.fetch("host"), + api_endpoint: @gitlab_client.endpoint, + repo: project.path_with_namespace, + directory: dependabot_config.directory, + branch: nil + ) + + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => Fetching dependency files..." + fetcher = Dependabot::FileFetchers.for_package_manager(package_manager).new( + source: source, + credentials: @available_credentials + ) + files = fetcher.files + commit = fetcher.commit + + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => Parsing dependencies information.." + parser = Dependabot::FileParsers.for_package_manager(package_manager).new( + dependency_files: files, + source: source, + credentials: @available_credentials + ) + + dependency_groups = create_dependency_groups(package_manager, project, dependabot_config.group_updates, parser.parse, files) + dependency_groups.each do |dependency_group| + get_updates_for_dependency_group(package_manager, project, dependabot_config, dependency_group, files) + open_merge_request = close_open_dependency_merge_requests(package_manager, project, dependency_group) + + next unless open_merge_request + + updated_dependencies = dependency_group. + dependencies.map(&:updates). + reject(&:empty?). + flatten + next if updated_dependencies.empty? + + updater = Dependabot::FileUpdaters.for_package_manager(package_manager).new( + dependencies: updated_dependencies, + dependency_files: files, + credentials: @available_credentials + ) + pr_creator = Dependabot::PullRequestCreator.new( + source: source, + base_commit: commit, + dependencies: updated_dependencies, + files: updater.updated_dependency_files, + credentials: @available_credentials, + commit_message_options: dependabot_config.commit_message, + label_language: true + ) + + # If the dependency group matches more than one package treat as a group + if dependency_group.dependencies.length > 1 + pr_creator.send(:branch_namer).instance_variable_set(:@name, dependency_group.branch_name) + end + + message_builder = MessageBuilder.new( + dependency_group_name: dependency_group.group_name, + source: pr_creator.source, + dependencies: pr_creator.dependencies, + files: pr_creator.files, + credentials: pr_creator.credentials, + commit_message_options: pr_creator.commit_message_options, + pr_message_header: pr_creator.pr_message_header, + pr_message_footer: pr_creator.pr_message_footer, + vulnerabilities_fixed: pr_creator.vulnerabilities_fixed, + github_redirection_service: pr_creator.github_redirection_service + ) + pr_creator.instance_variable_set(:@message_builder, message_builder) + pull_request = pr_creator.create + + unless pull_request + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => #{dependency_group.branch_name}) => Merge request already exists" + next + end + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => #{dependency_group.branch_name}) => Merge request created (#{pull_request.iid})" + + auto_merge = dependency_group.dependencies. + collect(&:auto_merge). + any? { |auto_merge| auto_merge } + next unless auto_merge + + # Wait for pipelines to be created - or give up + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => #{dependency_group.branch_name} => Waiting for pipelines to start" + pipelines = wait_for_pipelines_to_start(project, pull_request) + next if pipelines.nil? || pipelines.empty? + + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => #{dependency_group.branch_name} => Setting Merge request to auto-merge" + @gitlab_client.accept_merge_request( + project.id, + pull_request.iid, + merge_when_pipeline_succeeds: true, + should_remove_source_branch: true + ) + end + rescue StandardError => e + puts "#{@organisation} => #{project.path_with_namespace} => Failed processing: #{e.message}" + end + + def create_dependency_groups(package_manager, project, group_updates, dependencies, files) + groups = dependencies.select(&:top_level?).group_by do |dependency| + group_updates?(dependency, group_updates) + end + branch_namer = Dependabot::PullRequestCreator::BranchNamer.new(dependencies: dependencies, files: files, target_branch: nil) + groups.map do |group| + group_name = group[0] == "*" ? "" : group[0] + " " + branch_namer.instance_variable_set(:@name, (group[0] == "*" ? "dependencies" : group[0]).gsub(" ", "-")) + source_branch = branch_namer.new_branch_name + obj = { + group_name: group_name, + merge_request: merge_request_for_source_branch?(project, source_branch), + source_branch: source_branch, + branch_name: branch_namer.send(:sanitize_ref, + (group[0] == "*" ? "dependencies" : group[0]).gsub(" ", "-")), + dependencies: group[1]. + sort_by { |dependency| dependency.name }. + map { |dependency| OpenStruct.new({ package: dependency, updates: [], auto_merge: false }) } + } + OpenStruct.new(obj) + end + end + + def get_updates_for_dependency_group(package_manager, project, dependabot_config, dependency_group, files) + dependency_group.dependencies.each do |dependency| + if ignored_updates?(dependency.package, dependabot_config.ignored_updates) + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => #{dependency.package.name} (#{dependency.package.version}) => Dependency ignored" + next + end + + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => #{dependency.package.name} (#{dependency.package.version}) => Checking for updates.." + checker = Dependabot::UpdateCheckers.for_package_manager(package_manager).new( + dependency: dependency.package, + dependency_files: files, + credentials: @available_credentials + ) + + # Check if the dependency is up to date. + if checker.up_to_date? + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => #{dependency.package.name} (#{dependency.package.version}) => Already up to date" + next + end + + # Check if the dependency can be updated. + requirements_to_unlock = + if !checker.requirements_unlocked_or_can_be? + if checker.can_update?(requirements_to_unlock: :none) then :none + else :update_not_possible + end + elsif checker.can_update?(requirements_to_unlock: :own) then :own + elsif checker.can_update?(requirements_to_unlock: :all) then :all + else :update_not_possible + end + if requirements_to_unlock == :update_not_possible + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => #{dependency.package.name} (#{dependency.package.version}) => Cannot be updated" + next + end + dependency.updates = checker.updated_dependencies(requirements_to_unlock: requirements_to_unlock) + dependency.auto_merge = true if auto_merge_update(dependency.package, dependabot_config.automerged_updates) + end + end + + def wait_for_pipelines_to_start(project, pull_request) + max_retries = 3 + retry_count = 0 + begin + pipelines = @gitlab_client.merge_request_pipelines(project.id, pull_request.iid).auto_paginate + raise "Failed to merge request pipelines" if pipelines.nil? || pipelines.empty? + + pipelines + rescue StandardError + return nil if retry_count >= max_retries + + retry_count += 1 + puts "#{@organisation} => #{project.path_with_namespace} => Oh no, failed to get pipelines. Retries left: #{max_retries - retry_count}" + sleep(5) + retry + end + end + + def wait_while_pipelines_not_finish(project, pipelines) + pipelines.each do |pipeline| + obj = @gitlab_client.pipeline(project.id, pipeline.id) + next if %w(success failed canceled skipped).include?(obj.status) + + puts "#{@organisation} => #{project.path_with_namespace} => Pipeline #{obj.id} not yet complete #{obj.status}" + sleep(30) + redo + end + end + + def ignored_updates?(dependency, ignored_updates) + ignored_updates.each do |ignored_update| + return true if match_dependency?(dependency.name, [ignored_update.dependency_name]) + end + false + end + + def auto_merge_update(dependency, automerged_updates) + automerged_updates.each do |automerged_update| + return true if automerged_update.update_type == "all" + return true if match_dependency?(dependency.name, [automerged_update.dependency_name]) + end + false + end + + def group_updates?(dependency, group_updates) + group_updates.each do |group_update| + return group_update.dependency_name.capitalize if match_dependency?(dependency.name, [group_update.dependency_name]) + end + dependency.name + end + + def close_open_dependency_merge_requests(package_manager, project, dependency_group) + open_merge_requests = @gitlab_client. + merge_requests(project.id, { state: "opened" }). + select { |merge_request| merge_request.author.username == @merge_request_author } + + # Dependencies that should be updated + dependencies_updated = dependency_group. + dependencies.select { |dependency| dependency.updates.length >= 1 } + close_merge_requests = [] + open_merge_request = false + if !dependency_group.merge_request.nil? + unrelated_updates = dependencies_updated.empty? + dependencies_updated.each do |dependency| + # Close merge requests with matching package name; will be part of group + close_merge_requests += open_merge_requests.select { |merge_request| merge_request.title.include?(dependency.package.name) } + + if unrelated_to_merge_request?(dependency_group.merge_request, dependency.package, dependency.updates) + unrelated_updates = true + end + end + + # Close dependency group MR is we have unrelated package updates + close_open_merge_request(package_manager, project, dependency_group.merge_request) if unrelated_updates + open_merge_request = unrelated_updates + elsif dependencies_updated.length > 1 + # Close merge requests with matching package name; will be part of group + dependencies_updated.each do |dependency| + close_merge_requests += open_merge_requests.select { |merge_request| merge_request.title.include?(dependency.package.name) } + end + open_merge_request = true + elsif dependencies_updated.length == 1 + # Close merge requests with matching package name+version provided the new version is changing + close_merge_requests += open_merge_requests.select do |merge_request| + merge_request.title.include?(dependencies_updated[0].package.name) && + merge_request.title.include?(dependencies_updated[0].package.version) && + !merge_request.title.include?(dependencies_updated[0].updates.first.version) + end + open_merge_request = !close_merge_requests.empty? + end + + # Find open merge requests for dependencies NOT updated + dependency_group.dependencies. + select { |dependency| dependency.updates.empty? }. + each do |dependency| + close_merge_requests += open_merge_requests. + select { |merge_request| merge_request.title.include?(dependency.package.name) } + end + + # Close merge requests selected + close_merge_requests.each { |merge_request| close_open_merge_request(package_manager, project, merge_request) } + open_merge_request + end + + def close_open_merge_request(package_manager, project, merge_request) + return if merge_request.nil? + + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => Close merge request (#{merge_request.title})" + @gitlab_client.update_merge_request(project.path_with_namespace, merge_request.iid, { state_event: "close" }) + + puts "#{@organisation} => #{project.path_with_namespace} => #{package_manager} => Delete branch #{merge_request.source_branch})" + @gitlab_client.delete_branch(project.path_with_namespace, merge_request.source_branch) + end + + def merge_request_for_source_branch?(project, source_branch) + open_merge_requests = @gitlab_client.merge_requests(project.id, { state: "opened" }) + merge_requests = open_merge_requests.select do |merge_request| + merge_request.author.username == @merge_request_author && + merge_request.source_branch == source_branch + end + return nil if merge_requests.empty? + + merge_requests = merge_requests.collect do |merge_request| + hash = merge_request.to_h + hash["related_dependencies_updated"] = dependencies_from_merge_request?(merge_request) + return Gitlab::ObjectifiedHash.new(hash) + end + merge_requests.first + end + + def unrelated_to_merge_request?(merge_request, dependency, updated_dependencies) + return false if merge_request.nil? + + related_dependencies = merge_request.related_dependencies_updated + related_dependencies = related_dependencies.select { |related_dependency| related_dependency.include?(dependency.name) } + return true if related_dependencies.empty? # package not referenced + + related_dependencies = related_dependencies.select do |related_dependency| + related_dependency.include?(dependency.version) && + related_dependency.include?(updated_dependencies.first.version) + end + + related_dependencies.empty? # package version and new version not referenced + end + + def dependencies_from_merge_request?(merge_request) + return [] if merge_request.nil? + + merge_request. + description. + each_line(chomp: true). + select { |line| /^bumps .* from .* to .*\./i =~ line } + end + + def match_dependency?(dependency_name, dependency_name_matches) + match = dependency_name_matches.include? dependency_name + if match == false + match = dependency_name_matches. + select { |dependency_name_match| dependency_name_match.end_with? "*" }. + map { |dependency_name_match| dependency_name_match[0...-1] }. + map { |dependency_name_match| dependency_name.start_with? dependency_name_match }. + any? + end + match + end + + def package_manager(dependabot_config) + case dependabot_config.package_manager + when "javascript" + "npm_and_yarn" + when "ruby:bundler" + "bundler" + when "php:composer" + "composer" + when "python" + "pip" + when "go:modules" + "go_modules" + when "go:dep" + "dep" + when "java:maven" + "maven" + when "java:gradle" + "gradle" + when "dotnet:nuget" + "nuget" + when "rust:cargo" + "cargo" + when "elixir:hex" + "hex" + when "docker" + "docker" + when "terraform" + "terraform" + when "submodules" + "submodules" + when "elm" + "elm" + when "cake" + "cake" + else + raise "Unsupported package manager: #{dependabot_config.package_manager}" + end + end +end diff --git a/message_builder.rb b/message_builder.rb new file mode 100644 index 00000000..de704083 --- /dev/null +++ b/message_builder.rb @@ -0,0 +1,94 @@ +require "pathname" +require "dependabot/clients/github_with_retries" +require "dependabot/clients/gitlab_with_retries" +require "dependabot/metadata_finders" +require "dependabot/pull_request_creator" + +class MessageBuilder < Dependabot::PullRequestCreator::MessageBuilder + def initialize(dependency_group_name:, source:, dependencies:, files:, credentials:, + pr_message_header: nil, pr_message_footer: nil, + commit_message_options: {}, vulnerabilities_fixed: {}, + github_redirection_service: nil) + super(source: source, dependencies: dependencies, files: files, credentials: credentials, + pr_message_header: pr_message_header, pr_message_footer: pr_message_footer, + commit_message_options: commit_message_options, vulnerabilities_fixed: vulnerabilities_fixed, + github_redirection_service: github_redirection_service) + @dependency_group_name = dependency_group_name + end + + private + + def application_pr_name + if dependencies.count == 1 + super + elsif updating_a_property? + super + elsif updating_a_dependency_set? + super + else + pr_name = "bump " + pr_name = pr_name.capitalize if pr_name_prefixer.capitalize_first_word? + pr_name += "#{@dependency_group_name}dependencies" + end + end + + def requirement_commit_message_intro + msg = "Bumps the requirements on " + + msg += + if dependencies.count == 1 + "#{dependency_links.first} " + else + "#{dependency_links[0..-2].join(', ')} and #{dependency_links[-1]} " + end + + msg + "to permit the latest version." + end + + def multidependency_intro + "" + end + + def metadata_links + if dependencies.count == 1 + return metadata_links_for_dep(dependencies.first) + end + + dependencies.map do |dep| + "Bumps `#{dep.display_name}` from #{previous_version(dep)} to "\ + "#{new_version(dep)}."\ + "#{metadata_links_for_dep(dep)}" + end.join("\n\n") + end + + def metadata_cascades + if dependencies.one? + return metadata_cascades_for_dep(dependencies.first) + end + + dependencies.map do |dependency| + msg = "Bumps #{dependency_link(dependency)} "\ + "from #{previous_version(dependency)} "\ + "to #{new_version(dependency)}." + + if vulnerabilities_fixed[dependency.name]&.one? + msg += " **This update includes a security fix.**" + elsif vulnerabilities_fixed[dependency.name]&.any? + msg += " **This update includes security fixes.**" + end + + msg + metadata_cascades_for_dep(dependency) + end.join("\n\n") + end + + def dependency_link(dependency) + if source_url(dependency) + "[#{dependency.display_name}](#{source_url(dependency)})" + elsif homepage_url(dependency) + "[#{dependency.display_name}](#{homepage_url(dependency)})" + else + dependency.display_name + end + end + +end \ No newline at end of file diff --git a/run.rb b/run.rb new file mode 100644 index 00000000..04bab395 --- /dev/null +++ b/run.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true +# This script is designed to loop through all dependencies in a GHE, GitLab or +# Azure DevOps project, creating PRs where necessary. + +require_relative "gitlab-processor" + +# Capture environment variables. +organisation = ENV["GITLAB_ORGANIZATION"] || "Pharos" +credentials = [ + { + "type" => "git_source", + "host" => "github.com", + "username" => "x-access-token", + "password" => ENV["GITHUB_ACCESS_TOKEN"] # A GitHub access token with read access to public repos + }, + { + "type" => "git_source", + "host" => ENV["GITLAB_HOSTNAME"] || "git-us.pharos.com", + "username" => "x-access-token", + "password" => ENV["GITLAB_ACCESS_TOKEN"] + } +] + +unless ENV["LOCAL_CONFIG_VARIABLES"].to_s.strip.empty? + # For example: + # "[{\"type\":\"npm_registry\",\"registry\":\ + # "registry.npmjs.org\",\"token\":\"123\"}]" + credentials.concat(JSON.parse(ENV["LOCAL_CONFIG_VARIABLES"])) +end + +# Full name of the repo you want to create pull requests for. +repository_name = ENV["PROJECT_PATH"] # namespace/project +mr_limit_per_repo = ENV["MR_LIMIT_PER_REPO"].to_i || 5 # namespace/project + +GitLabProcessor.new(organisation, credentials).process(repository_name, mr_limit_per_repo)