diff --git a/cucumber.yml b/cucumber.yml
index 1660e80ccb..e4bd3cf8ec 100644
--- a/cucumber.yml
+++ b/cucumber.yml
@@ -13,9 +13,9 @@ end
 wip_opts = "--color --tags @wip:3"
 wip_opts << ",@wip-jruby:3" if defined?(JRUBY_VERSION)
 %>
-default: <%= std_opts %> --tags ~@jruby --dotcucumber features/.cucumber
-jruby: <%= std_opts %> --tags ~@spork --tags ~@wire
-jruby_win: <%= std_opts %> --tags ~@spork --tags ~@wire CUCUMBER_FORWARD_SLASH_PATHS=true
+default:     <%= std_opts %> --tags ~@jruby --dotcucumber features/.cucumber
+jruby:       <%= std_opts %> --tags ~@spork --tags ~@wire --tags ~@not-jruby
+jruby_win:   <%= std_opts %> --tags ~@spork --tags ~@wire --tags ~@not-jruby CUCUMBER_FORWARD_SLASH_PATHS=true
 windows_mri: <%= std_opts %> --tags ~@jruby --tags ~@spork --tags ~@wire --tags ~@needs-many-fonts CUCUMBER_FORWARD_SLASH_PATHS=true
 ruby_1_9: <%= std_opts %> --tags ~@jruby --tags ~@fails_on_1_9
 ruby_2_0: <%= std_opts %> --tags ~@jruby --tags ~@fails_on_1_9
diff --git a/features/backtraces.feature b/features/backtraces.feature
index 4180110277..c9163ad1a7 100644
--- a/features/backtraces.feature
+++ b/features/backtraces.feature
@@ -34,3 +34,37 @@ Feature: Backtraces
             java.util.AbstractList.add(AbstractList.java:108)
       """
 
+  @not-jruby
+  Scenario: Ruby backtraces
+    Given a file named "game.rb" with:
+      """
+      class Game
+        def start; end
+      end
+      """
+    And a file named "features/start.feature" with:
+      """
+      Feature:
+        Scenario:
+          When I start the game
+      """
+    And a file named "features/step_definitions/steps.rb" with:
+      """
+      require File.dirname(__FILE__) + '/../../game'
+      When /start/ do
+        Game.new.start('a spurious argument')
+      end
+      """
+    When I run `cucumber`
+    Then it should fail with:
+      """
+      Feature: 
+      
+        Scenario:               # features/start.feature:2
+          When I start the game # features/step_definitions/steps.rb:2
+            wrong number of arguments (1 for 0) (ArgumentError)
+            ./game.rb:2:in `start'
+            ./features/step_definitions/steps.rb:3:in `/start/'
+            features/start.feature:3:in `When I start the game'
+      """
+