Merge pull request #3230 from dependabot/brrygrdn/add-experimental-fl…

…ag-for-bundler-2

Add an options argument to the bundler_version helper method that is passed from the public API down

.github/workflows/ci.yml

@@ -107,6 +107,11 @@ jobs:
         run: |
           docker run --rm "$CORE_CI_IMAGE" bash -c \
             "cd /home/dependabot/dependabot-core/bundler/helpers/v1 && BUNDLER_VERSION=1 bundle install && BUNDLER_VERSION=1 bundle exec rspec spec"
+      - name: Run bundler v2 native helper specs
+        if: matrix.suite == 'bundler'
+        run: |
+          docker run --rm "$CORE_CI_IMAGE" bash -c \
+            "cd /home/dependabot/dependabot-core/bundler/helpers/v2 && BUNDLER_VERSION=2 bundle install && BUNDLER_VERSION=2 bundle exec rspec spec"
       - name: Run ${{ matrix.suite }} tests with rspec
         run: |
           docker run --env "CI=true" --env "DEPENDABOT_TEST_ACCESS_TOKEN=$DEPENDABOT_TEST_ACCESS_TOKEN" --rm "$CORE_CI_IMAGE" bash -c \

Dockerfile

@@ -196,6 +196,7 @@ RUN bash /opt/terraform/helpers/build /opt/terraform && \
   bash /opt/python/helpers/build /opt/python && \
   bash /opt/dep/helpers/build /opt/dep && \
   mkdir -p /opt/bundler/v1 && bash /opt/bundler/helpers/v1/build /opt/bundler/v1 && \
+  mkdir -p /opt/bundler/v2 && bash /opt/bundler/helpers/v2/build /opt/bundler/v2 && \
   bash /opt/go_modules/helpers/build /opt/go_modules && \
   bash /opt/npm_and_yarn/helpers/build /opt/npm_and_yarn && \
   bash /opt/hex/helpers/build /opt/hex && \

bundler/helpers/v2/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_PATH: ".bundle"

bundler/helpers/v2/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/*
+!/.bundle/config
+/.env
+/tmp
+/dependabot-*.gem
+Gemfile.lock
+spec/fixtures/projects/*/.bundle/
+!spec/fixtures/projects/**/Gemfile.lock
+!spec/fixtures/projects/**/vendor

bundler/helpers/v2/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# NOTE: Used to run native helper specs
+group :test do
+  gem "byebug", "11.1.3"
+  gem "rspec", "3.10.0"
+  gem "rspec-its", "1.3.0"
+  gem "vcr", "6.0.0"
+  gem "webmock", "3.12.1"
+end

bundler/helpers/v2/build

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+install_dir=$1
+if [ -z "$install_dir" ]; then
+  echo "usage: $0 INSTALL_DIR"
+  exit 1
+fi
+
+helpers_dir="$(dirname "${BASH_SOURCE[0]}")"
+cp -r \
+  "$helpers_dir/.bundle" \
+  "$helpers_dir/lib" \
+  "$helpers_dir/run.rb" \
+  "$helpers_dir/Gemfile" \
+  "$install_dir"
+
+cd "$install_dir"
+
+# NOTE: Sets `BUNDLED WITH` to match the installed v1 version in Gemfile.lock
+# forcing specs and native helpers to run with the same version
+BUNDLER_VERSION=2 bundle install

bundler/helpers/v2/lib/functions.rb

@@ -0,0 +1,67 @@
+module Functions
+  class NotImplementedError < StandardError; end
+
+  def self.parsed_gemfile(lockfile_name:, gemfile_name:, dir:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.parsed_gemspec(lockfile_name:, gemspec_name:, dir:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.vendor_cache_dir(dir:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.update_lockfile(dir:, gemfile_name:, lockfile_name:, using_bundler2:,
+                           credentials:, dependencies:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.force_update(dir:, dependency_name:, target_version:, gemfile_name:,
+                        lockfile_name:, using_bundler2:, credentials:,
+                        update_multiple_dependencies:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.dependency_source_type(gemfile_name:, dependency_name:, dir:,
+                                  credentials:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.depencency_source_latest_git_version(gemfile_name:, dependency_name:,
+                                                dir:, credentials:,
+                                                dependency_source_url:,
+                                                dependency_source_branch:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.private_registry_versions(gemfile_name:, dependency_name:, dir:,
+                                     credentials:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.resolve_version(dependency_name:, dependency_requirements:,
+                           gemfile_name:, lockfile_name:, using_bundler2:,
+                           dir:, credentials:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.jfrog_source(dir:, gemfile_name:, credentials:, using_bundler2:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.git_specs(dir:, gemfile_name:, credentials:, using_bundler2:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.set_bundler_flags_and_credentials(dir:, credentials:,
+                                             using_bundler2:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+
+  def self.conflicting_dependencies(dir:, dependency_name:, target_version:,
+                                    lockfile_name:, using_bundler2:, credentials:)
+    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+  end
+end

bundler/helpers/v2/run.rb

@@ -0,0 +1,30 @@
+require "bundler"
+require "json"
+
+$LOAD_PATH.unshift(File.expand_path("./lib", __dir__))
+$LOAD_PATH.unshift(File.expand_path("../v1/monkey_patches", __dir__))
+
+# Bundler monkey patches
+require "definition_ruby_version_patch"
+require "definition_bundler_version_patch"
+require "git_source_patch"
+
+require "functions"
+
+def output(obj)
+  print JSON.dump(obj)
+end
+
+begin
+  request = JSON.parse($stdin.read)
+
+  function = request["function"]
+  args = request["args"].transform_keys(&:to_sym)
+
+  output({ result: Functions.send(function, **args) })
+rescue => error
+  output(
+    { error: error.message, error_class: error.class, trace: error.backtrace }
+  )
+  exit(1)
+end

bundler/helpers/v2/spec/functions_spec.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "native_spec_helper"
+
+RSpec.describe Functions do
+  # Verify v1 method signatures are exist, but raise as NYI
+  {
+    parsed_gemfile: [ :lockfile_name, :gemfile_name, :dir ],
+    parsed_gemspec: [ :lockfile_name, :gemspec_name, :dir ],
+    vendor_cache_dir: [ :dir ],
+    update_lockfile: [ :dir, :gemfile_name, :lockfile_name, :using_bundler2, :credentials, :dependencies ],
+    force_update: [ :dir, :dependency_name, :target_version, :gemfile_name, :lockfile_name, :using_bundler2,
+                    :credentials, :update_multiple_dependencies ],
+    dependency_source_type: [ :gemfile_name, :dependency_name, :dir, :credentials ],
+    depencency_source_latest_git_version: [ :gemfile_name, :dependency_name, :dir, :credentials, :dependency_source_url,
+                                            :dependency_source_branch  ],
+    private_registry_versions: [:gemfile_name, :dependency_name, :dir, :credentials ],
+    resolve_version: [:dependency_name, :dependency_requirements, :gemfile_name, :lockfile_name, :using_bundler2,
+                      :dir, :credentials],
+    jfrog_source: [:dir, :gemfile_name, :credentials, :using_bundler2],
+    git_specs: [:dir, :gemfile_name, :credentials, :using_bundler2],
+    set_bundler_flags_and_credentials: [:dir, :credentials, :using_bundler2],
+    conflicting_dependencies: [:dir, :dependency_name, :target_version, :lockfile_name, :using_bundler2, :credentials]
+  }.each do |function, kwargs|
+    describe "::#{function}" do
+      let(:args) do
+        kwargs.inject({}) do |args, keyword|
+          args.merge({ keyword => anything })
+        end
+      end
+
+      it "raises a NYI" do
+        expect { Functions.send(function, **args) }.to raise_error(Functions::NotImplementedError)
+      end
+    end
+  end
+end

bundler/helpers/v2/spec/native_spec_helper.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "rspec/its"
+require "webmock/rspec"
+require "byebug"
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+# TODO: Fork `v1/monkey_patches` into `v2/monkey_patches` ?
+$LOAD_PATH.unshift(File.expand_path("../../v1/monkey_patches", __dir__))
+
+# Bundler monkey patches
+require "definition_ruby_version_patch"
+require "definition_bundler_version_patch"
+require "git_source_patch"
+
+require "functions"
+
+RSpec.configure do |config|
+  config.color = true
+  config.order = :rand
+  config.mock_with(:rspec) { |mocks| mocks.verify_partial_doubles = true }
+  config.raise_errors_for_deprecations!
+end
+
+# Duplicated in lib/dependabot/bundler/file_updater/lockfile_updater.rb
+# TODO: Stop sanitizing the lockfile once we have bundler 2 installed
+LOCKFILE_ENDING = /(?<ending>\s*(?:RUBY VERSION|BUNDLED WITH).*)/m.freeze
+
+def project_dependency_files(project)
+  project_path = File.expand_path(File.join("../../spec/fixtures/projects/bundler1", project))
+  Dir.chdir(project_path) do
+    # NOTE: Include dotfiles (e.g. .npmrc)
+    files = Dir.glob("**/*", File::FNM_DOTMATCH)
+    files = files.select { |f| File.file?(f) }
+    files.map do |filename|
+      content = File.read(filename)
+      if filename == "Gemfile.lock"
+        content = content.gsub(LOCKFILE_ENDING, "")
+      end
+      {
+        name: filename,
+        content: content
+      }
+    end
+  end
+end
+
+def fixture(*name)
+  File.read(File.join("../../spec/fixtures", File.join(*name)))
+end

bundler/lib/dependabot/bundler/file_parser.rb

@@ -301,7 +301,7 @@ def imported_ruby_files
       end
 
       def bundler_version
-        @bundler_version ||= Helpers.bundler_version(lockfile)
+        @bundler_version ||= Helpers.bundler_version(lockfile, options: options)
       end
     end
   end

bundler/lib/dependabot/bundler/file_updater.rb

@@ -151,7 +151,8 @@ def updated_lockfile_content
             dependencies: dependencies,
             dependency_files: dependency_files,
             repo_contents_path: repo_contents_path,
-            credentials: credentials
+            credentials: credentials,
+            options: options
           ).updated_lockfile_content
       end
 
@@ -162,7 +163,7 @@ def top_level_gemspecs
       end
 
       def bundler_version
-        @bundler_version ||= Helpers.bundler_version(lockfile)
+        @bundler_version ||= Helpers.bundler_version(lockfile, options: options)
       end
     end
   end

bundler/lib/dependabot/bundler/file_updater/lockfile_updater.rb

@@ -33,11 +33,12 @@ def gemspec_sources
         end
 
         def initialize(dependencies:, dependency_files:,
-                       repo_contents_path: nil, credentials:)
+                       repo_contents_path: nil, credentials:, options:)
           @dependencies = dependencies
           @dependency_files = dependency_files
           @repo_contents_path = repo_contents_path
           @credentials = credentials
+          @options = options
         end
 
         def updated_lockfile_content
@@ -54,7 +55,7 @@ def updated_lockfile_content
         private
 
         attr_reader :dependencies, :dependency_files, :repo_contents_path,
-                    :credentials
+                    :credentials, :options
 
         def build_updated_lockfile
           base_dir = dependency_files.first.directory
@@ -304,7 +305,7 @@ def using_bundler2?
         end
 
         def bundler_version
-          @bundler_version ||= Helpers.bundler_version(lockfile)
+          @bundler_version ||= Helpers.bundler_version(lockfile, options: options)
         end
       end
     end

bundler/lib/dependabot/bundler/helpers.rb

@@ -6,9 +6,14 @@ module Helpers
       V1 = "1"
       V2 = "2"
 
-      # TODO: Add support for bundler v2
-      # return "v2" if lockfile.content.match?(/BUNDLED WITH\s+2/m)
-      def self.bundler_version(_lockfile)
+      # NOTE: options is a manditory argument to ensure we pass it from all calling classes
+      def self.bundler_version(_lockfile, options:)
+        # For now, force V2 if bundler_2_available
+        return V2 if options[:bundler_2_available]
+
+        # TODO: Add support for bundler v2 based on lockfile
+        # return V2 if lockfile.content.match?(/BUNDLED WITH\s+2/m)
+
         V1
       end
     end

bundler/lib/dependabot/bundler/native_helpers.rb

@@ -22,6 +22,11 @@ def self.run_bundler_subprocess(function:, args:, bundler_version:)
               "GEM_HOME" => File.join(versioned_helper_path(bundler_version: bundler_version), ".bundle")
             }
           )
+        rescue SharedHelpers::HelperSubprocessFailed => e
+          # TODO: Remove once we stop stubbing out the V2 native helper
+          raise Dependabot::NotImplemented, e.message if e.error_class == "Functions::NotImplementedError"
+
+          raise
         end
       end