Redo: Run Bundler v1 helpers with explicit version

This brings back this revert:
#3222

We've wrapped bundler native helpers in `Bundler.with_original_env` to
use the env before bundler was initialised in core.

This removed the need to manually unset env vars like `GEM_HOME` and
`GEM_PATH` which caused bundler to default git install to
`/var/lib/gems/2.6.0/cache/bundler/git` when run in the parent
container, where it doesn't have write permission.

With this change the the bundle cache is written to the native helper
folder/.bundle.

Co-authored-by: Barry Gordon <brrygrdn@github.com>

Dockerfile

@@ -194,7 +194,7 @@ ENV DEPENDABOT_NATIVE_HELPERS_PATH="/opt" \
 RUN bash /opt/terraform/helpers/build /opt/terraform && \
   bash /opt/python/helpers/build /opt/python && \
   bash /opt/dep/helpers/build /opt/dep && \
-  bash /opt/bundler/helpers/build /opt/bundler && \
+  mkdir -p /opt/bundler/v1 && bash /opt/bundler/helpers/v1/build /opt/bundler/v1 && \
   bash /opt/go_modules/helpers/build /opt/go_modules && \
   bash /opt/npm_and_yarn/helpers/build /opt/npm_and_yarn && \
   bash /opt/hex/helpers/build /opt/hex && \

bundler/helpers/v1/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_PATH: ".bundle"

bundler/helpers/v1/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/*
+!/.bundle/config
+/.env
+/tmp
+/dependabot-*.gem
+Gemfile.lock
+spec/fixtures/projects/*/.bundle/
+!spec/fixtures/projects/**/Gemfile.lock
+!spec/fixtures/projects/**/vendor

bundler/helpers/v1/Gemfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# NOTE: This is intentionally left blank as it's currently only used to force
+# bundler to use v1 when executing native helpers by pointing the BUNDLE_GEMFILE
+# env to this Gemfile in Dependabot::Bundler::NativeHelpers

bundler/helpers/build → bundler/helpers/v1/build

@@ -10,9 +10,15 @@ fi
 
 helpers_dir="$(dirname "${BASH_SOURCE[0]}")"
 cp -r \
+  "$helpers_dir/.bundle" \
   "$helpers_dir/lib" \
   "$helpers_dir/monkey_patches" \
   "$helpers_dir/run.rb" \
+  "$helpers_dir/Gemfile" \
   "$install_dir"
 
 cd "$install_dir"
+
+# NOTE: Sets `BUNDLED WITH` to match the installed v1 version in Gemfile.lock
+# forcing specs and native helpers to run with the same version
+BUNDLER_VERSION=1 bundle install

bundler/lib/dependabot/bundler/file_parser.rb

@@ -5,6 +5,7 @@
 require "dependabot/file_parsers/base"
 require "dependabot/bundler/file_updater/lockfile_updater"
 require "dependabot/bundler/native_helpers"
+require "dependabot/bundler/helpers"
 require "dependabot/bundler/version"
 require "dependabot/shared_helpers"
 require "dependabot/errors"
@@ -129,8 +130,8 @@ def parsed_gemfile
                                                       repo_contents_path) do
             write_temporary_dependency_files
 
-            SharedHelpers.run_helper_subprocess(
-              command: NativeHelpers.helper_path,
+            NativeHelpers.run_bundler_subprocess(
+              bundler_version: bundler_version,
               function: "parsed_gemfile",
               args: {
                 gemfile_name: gemfile.name,
@@ -159,8 +160,8 @@ def parsed_gemspec(file)
                                                       repo_contents_path) do
             write_temporary_dependency_files
 
-            SharedHelpers.run_helper_subprocess(
-              command: NativeHelpers.helper_path,
+            NativeHelpers.run_bundler_subprocess(
+              bundler_version: bundler_version,
               function: "parsed_gemspec",
               args: {
                 gemspec_name: file.name,
@@ -298,6 +299,10 @@ def imported_ruby_files
           select { |f| f.name.end_with?(".rb") }.
           reject { |f| f.name == "gems.rb" }
       end
+
+      def bundler_version
+        @bundler_version ||= Helpers.bundler_version(lockfile)
+      end
     end
   end
 end

bundler/lib/dependabot/bundler/file_updater.rb

@@ -3,6 +3,7 @@
 require "dependabot/file_updaters"
 require "dependabot/file_updaters/base"
 require "dependabot/bundler/native_helpers"
+require "dependabot/bundler/helpers"
 require "dependabot/file_updaters/vendor_updater"
 
 module Dependabot
@@ -75,8 +76,8 @@ def vendor_cache_dir
         return @vendor_cache_dir if defined?(@vendor_cache_dir)
 
         @vendor_cache_dir =
-          SharedHelpers.run_helper_subprocess(
-            command: NativeHelpers.helper_path,
+          NativeHelpers.run_bundler_subprocess(
+            bundler_version: bundler_version,
             function: "vendor_cache_dir",
             args: {
               dir: repo_contents_path
@@ -159,6 +160,10 @@ def top_level_gemspecs
           select { |file| file.name.end_with?(".gemspec") }.
           reject(&:support_file?)
       end
+
+      def bundler_version
+        @bundler_version ||= Helpers.bundler_version(lockfile)
+      end
     end
   end
 end

bundler/lib/dependabot/bundler/file_updater/lockfile_updater.rb

@@ -6,6 +6,7 @@
 require "dependabot/errors"
 require "dependabot/bundler/file_updater"
 require "dependabot/bundler/native_helpers"
+require "dependabot/bundler/helpers"
 
 module Dependabot
   module Bundler
@@ -64,8 +65,8 @@ def build_updated_lockfile
             ) do |tmp_dir|
               write_temporary_dependency_files
 
-              SharedHelpers.run_helper_subprocess(
-                command: NativeHelpers.helper_path,
+              NativeHelpers.run_bundler_subprocess(
+                bundler_version: bundler_version,
                 function: "update_lockfile",
                 args: {
                   gemfile_name: gemfile.name,
@@ -301,6 +302,10 @@ def using_bundler2?
 
           lockfile.content.match?(/BUNDLED WITH\s+2/m)
         end
+
+        def bundler_version
+          @bundler_version ||= Helpers.bundler_version(lockfile)
+        end
       end
     end
   end

bundler/lib/dependabot/bundler/helpers.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Dependabot
+  module Bundler
+    module Helpers
+      V1 = "1"
+      V2 = "2"
+
+      # TODO: Add support for bundler v2
+      # return "v2" if lockfile.content.match?(/BUNDLED WITH\s+2/m)
+      def self.bundler_version(_lockfile)
+        V1
+      end
+    end
+  end
+end

bundler/lib/dependabot/bundler/native_helpers.rb

@@ -1,10 +1,35 @@
 # frozen_string_literal: true
 
+require "bundler"
+require "dependabot/shared_helpers"
+
 module Dependabot
   module Bundler
     module NativeHelpers
-      def self.helper_path
-        "bundle exec ruby #{File.join(native_helpers_root, 'run.rb')}"
+      def self.run_bundler_subprocess(function:, args:, bundler_version:)
+        # Run helper suprocess with all bundler-related ENV variables removed
+        ::Bundler.with_original_env do
+          SharedHelpers.run_helper_subprocess(
+            command: helper_path(bundler_version: bundler_version),
+            function: function,
+            args: args,
+            env: {
+              # Bundler will pick the matching installed major version
+              "BUNDLER_VERSION" => bundler_version,
+              "BUNDLE_GEMFILE" => File.join(versioned_helper_path(bundler_version: bundler_version), "Gemfile"),
+              "BUNDLE_PATH" => File.join(versioned_helper_path(bundler_version: bundler_version), ".bundle")
+            }
+          )
+        end
+      end
+
+      def self.versioned_helper_path(bundler_version:)
+        native_helper_version = "v#{bundler_version}"
+        File.join(native_helpers_root, native_helper_version)
+      end
+
+      def self.helper_path(bundler_version:)
+        "ruby #{File.join(versioned_helper_path(bundler_version: bundler_version), 'run.rb')}"
       end
 
       def self.native_helpers_root

bundler/lib/dependabot/bundler/update_checker/conflicting_dependency_resolver.rb

@@ -2,6 +2,7 @@
 
 require "dependabot/bundler/update_checker"
 require "dependabot/bundler/native_helpers"
+require "dependabot/bundler/helpers"
 require "dependabot/shared_helpers"
 
 module Dependabot
@@ -28,8 +29,8 @@ def initialize(dependency_files:, repo_contents_path:, credentials:)
         #   * requirement [String] the requirement on the target_dependency
         def conflicting_dependencies(dependency:, target_version:)
           in_a_native_bundler_context(error_handling: false) do |tmp_dir|
-            SharedHelpers.run_helper_subprocess(
-              command: NativeHelpers.helper_path,
+            NativeHelpers.run_bundler_subprocess(
+              bundler_version: bundler_version,
               function: "conflicting_dependencies",
               args: {
                 dir: tmp_dir,
@@ -42,6 +43,12 @@ def conflicting_dependencies(dependency:, target_version:)
             )
           end
         end
+
+        private
+
+        def bundler_version
+          @bundler_version ||= Helpers.bundler_version(lockfile)
+        end
       end
     end
   end

bundler/lib/dependabot/bundler/update_checker/force_updater.rb

@@ -3,6 +3,7 @@
 require "dependabot/bundler/file_parser"
 require "dependabot/bundler/file_updater/lockfile_updater"
 require "dependabot/bundler/native_helpers"
+require "dependabot/bundler/helpers"
 require "dependabot/bundler/update_checker"
 require "dependabot/bundler/update_checker/requirements_updater"
 require "dependabot/errors"
@@ -43,8 +44,8 @@ def update_multiple_dependencies?
 
         def force_update
           in_a_native_bundler_context(error_handling: false) do |tmp_dir|
-            updated_deps, specs = SharedHelpers.run_helper_subprocess(
-              command: NativeHelpers.helper_path,
+            updated_deps, specs = NativeHelpers.run_bundler_subprocess(
+              bundler_version: bundler_version,
               function: "force_update",
               args: {
                 dir: tmp_dir,
@@ -146,6 +147,10 @@ def using_bundler2?
 
           lockfile.content.match?(/BUNDLED WITH\s+2/m)
         end
+
+        def bundler_version
+          @bundler_version ||= Helpers.bundler_version(lockfile)
+        end
       end
     end
   end

bundler/lib/dependabot/bundler/update_checker/latest_version_finder/dependency_source.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "dependabot/bundler/native_helpers"
+require "dependabot/bundler/helpers"
+
 module Dependabot
   module Bundler
     class UpdateChecker
@@ -53,8 +56,8 @@ def latest_git_version_details
 
             SharedHelpers.with_git_configured(credentials: credentials) do
               in_a_native_bundler_context do |tmp_dir|
-                SharedHelpers.run_helper_subprocess(
-                  command: NativeHelpers.helper_path,
+                NativeHelpers.run_bundler_subprocess(
+                  bundler_version: bundler_version,
                   function: "depencency_source_latest_git_version",
                   args: {
                     dir: tmp_dir,
@@ -98,8 +101,8 @@ def dependency_rubygems_uri
           def private_registry_versions
             @private_registry_versions ||=
               in_a_native_bundler_context do |tmp_dir|
-                SharedHelpers.run_helper_subprocess(
-                  command: NativeHelpers.helper_path,
+                NativeHelpers.run_bundler_subprocess(
+                  bundler_version: bundler_version,
                   function: "private_registry_versions",
                   args: {
                     dir: tmp_dir,
@@ -118,8 +121,8 @@ def source_type
             return @source_type = RUBYGEMS unless gemfile
 
             @source_type = in_a_native_bundler_context do |tmp_dir|
-              SharedHelpers.run_helper_subprocess(
-                command: NativeHelpers.helper_path,
+              NativeHelpers.run_bundler_subprocess(
+                bundler_version: bundler_version,
                 function: "dependency_source_type",
                 args: {
                   dir: tmp_dir,
@@ -135,6 +138,15 @@ def gemfile
             dependency_files.find { |f| f.name == "Gemfile" } ||
               dependency_files.find { |f| f.name == "gems.rb" }
           end
+
+          def lockfile
+            dependency_files.find { |f| f.name == "Gemfile.lock" } ||
+              dependency_files.find { |f| f.name == "gems.locked" }
+          end
+
+          def bundler_version
+            @bundler_version ||= Helpers.bundler_version(lockfile)
+          end
         end
       end
     end

bundler/lib/dependabot/bundler/update_checker/shared_bundler_helpers.rb

@@ -4,6 +4,7 @@
 
 require "dependabot/bundler/update_checker"
 require "dependabot/bundler/native_helpers"
+require "dependabot/bundler/helpers"
 require "dependabot/shared_helpers"
 require "dependabot/errors"
 
@@ -163,8 +164,8 @@ def handle_bundler_errors(error)
 
         def inaccessible_git_dependencies
           in_a_native_bundler_context(error_handling: false) do |tmp_dir|
-            git_specs = SharedHelpers.run_helper_subprocess(
-              command: NativeHelpers.helper_path,
+            git_specs = NativeHelpers.run_bundler_subprocess(
+              bundler_version: bundler_version,
               function: "git_specs",
               args: {
                 dir: tmp_dir,
@@ -187,8 +188,8 @@ def inaccessible_git_dependencies
 
         def jfrog_source
           in_a_native_bundler_context(error_handling: false) do |dir|
-            SharedHelpers.run_helper_subprocess(
-              command: NativeHelpers.helper_path,
+            NativeHelpers.run_bundler_subprocess(
+              bundler_version: bundler_version,
               function: "jfrog_source",
               args: {
                 dir: dir,
@@ -236,6 +237,10 @@ def using_bundler2?
 
           lockfile.content.match?(/BUNDLED WITH\s+2/m)
         end
+
+        def bundler_version
+          @bundler_version ||= Helpers.bundler_version(lockfile)
+        end
       end
     end
   end

bundler/lib/dependabot/bundler/update_checker/version_resolver.rb

@@ -2,6 +2,7 @@
 
 require "excon"
 
+require "dependabot/bundler/helpers"
 require "dependabot/bundler/update_checker"
 require "dependabot/bundler/file_updater/lockfile_updater"
 require "dependabot/bundler/requirement"
@@ -75,8 +76,8 @@ def fetch_latest_resolvable_version_details
             # some errors we want to handle specifically ourselves, including
             # potentially retrying in the case of the Ruby version being locked
             in_a_native_bundler_context(error_handling: false) do |tmp_dir|
-              details =  SharedHelpers.run_helper_subprocess(
-                command: NativeHelpers.helper_path,
+              details = NativeHelpers.run_bundler_subprocess(
+                bundler_version: bundler_version,
                 function: "resolve_version",
                 args: {
                   dependency_name: dependency.name,
@@ -218,6 +219,10 @@ def using_bundler2?
 
           lockfile.content.match?(/BUNDLED WITH\s+2/m)
         end
+
+        def bundler_version
+          @bundler_version ||= Helpers.bundler_version(lockfile)
+        end
       end
     end
   end