diff --git a/bundler/lib/bundler/shared_helpers.rb b/bundler/lib/bundler/shared_helpers.rb
index 405ade95ddfa..846b9cc3aa43 100644
--- a/bundler/lib/bundler/shared_helpers.rb
+++ b/bundler/lib/bundler/shared_helpers.rb
@@ -313,12 +313,11 @@ def bundler_ruby_lib
     end
 
     def clean_load_path
-      bundler_lib = bundler_ruby_lib
-
       loaded_gem_paths = Bundler.rubygems.loaded_gem_paths
 
       $LOAD_PATH.reject! do |p|
-        next if resolve_path(p).start_with?(bundler_lib)
+        resolved_path = resolve_path(p)
+        next if $LOADED_FEATURES.any? {|lf| lf.start_with?(resolved_path) }
         loaded_gem_paths.delete(p)
       end
       $LOAD_PATH.uniq!
diff --git a/bundler/spec/runtime/setup_spec.rb b/bundler/spec/runtime/setup_spec.rb
index 804e29c3c13a..35873dcaa967 100644
--- a/bundler/spec/runtime/setup_spec.rb
+++ b/bundler/spec/runtime/setup_spec.rb
@@ -1511,5 +1511,28 @@ def lock_with(ruby_version = nil)
 
       expect(out).to include("rack, yard")
     end
+
+    it "does not cause double loads when higher versions of default gems are activated before bundler" do
+      build_repo2 do
+        build_gem "json", "999.999.999" do |s|
+          s.write "lib/json.rb", <<~RUBY
+            module JSON
+              VERSION = "999.999.999"
+            end
+          RUBY
+        end
+      end
+
+      system_gems "json-999.999.999", :gem_repo => gem_repo2
+
+      install_gemfile "source \"#{file_uri_for(gem_repo1)}\""
+      ruby <<-RUBY
+        require "json"
+        require "bundler/setup"
+        require "json"
+      RUBY
+
+      expect(err).to be_empty
+    end
   end
 end