diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
index 8245eb3c6..4d366974f 100644
--- a/.github/workflows/ruby.yml
+++ b/.github/workflows/ruby.yml
@@ -37,6 +37,14 @@ jobs:
       - uses: actions/checkout@v2
 
       - uses: ruby/setup-ruby@v1
+        if: matrix.ruby >= '2.6'
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+          rubygems: latest
+
+      - uses: ruby/setup-ruby@v1
+        if: matrix.ruby < '2.6'  # Skip updating RubyGems on old rubies
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true