diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml
index d3315c00c5..388c4ee921 100644
--- a/.vsts-dotnet-ci.yml
+++ b/.vsts-dotnet-ci.yml
@@ -27,7 +27,8 @@ jobs:
         _config_short: RI
         _includeBenchmarkData: true
         _targetFramework: netcoreapp3.1
-    innerLoop: true
+    innerLoop: false
+    runSpecific: true
     pool:
       name: Hosted Ubuntu 1604
 
@@ -36,7 +37,8 @@ jobs:
     name: Ubuntu_x64_NetCoreApp21
     buildScript: ./build.sh
     container: UbuntuContainer
-    innerLoop: true
+    innerLoop: false
+    runSpecific: true
     pool:
       name: Hosted Ubuntu 1604
 
@@ -44,7 +46,8 @@ jobs:
   parameters:
     name: MacOS_x64_NetCoreApp21
     buildScript: ./build.sh
-    innerLoop: true
+    innerLoop: false
+    runSpecific: true
     pool:
       name: Hosted macOS
 
@@ -63,7 +66,8 @@ jobs:
         _config_short: RI
         _includeBenchmarkData: true
         _targetFramework: netcoreapp3.1
-    innerLoop: true
+    innerLoop: false
+    runSpecific: true
     vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0"
     pool:
       name: Hosted VS2017
@@ -72,7 +76,8 @@ jobs:
   parameters:
     name: Windows_x64_NetCoreApp21
     buildScript: build.cmd
-    innerLoop: true
+    innerLoop: false
+    runSpecific: true
     vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1"
     pool:
       name: Hosted VS2017
@@ -92,7 +97,8 @@ jobs:
         _config_short: RFX
         _includeBenchmarkData: false
         _targetFramework: win-x64
-    innerLoop: true
+    innerLoop: false
+    runSpecific: true
     vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0"
     pool:
       name: Hosted VS2017
@@ -102,7 +108,8 @@ jobs:
     name: Windows_x86_NetCoreApp21
     architecture: x86
     buildScript: build.cmd
-    innerLoop: true
+    innerLoop: false
+    runSpecific: true
     vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1"
     pool:
       name: Hosted VS2017
diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs
index 68dee8a5b9..4a9bd96d87 100644
--- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs
+++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs
@@ -7,6 +7,7 @@
 using System.IO;
 using System.Linq;
 using Microsoft.ML.Data;
+using Microsoft.ML.TestFrameworkCommon.Attributes;
 using Microsoft.ML.Trainers.FastTree;
 using Xunit;
 
@@ -420,9 +421,11 @@ public void TestFastForestBinaryFeaturizationInPipeline()
             Assert.True(metrics.AreaUnderPrecisionRecallCurve > 0.98);
         }
 
-        [Fact]
-        public void TestFastTreeRegressionFeaturizationInPipeline()
+        [Theory, IterationData(25)]
+        [TestCategory("RunSpecificTest")]
+        public void TestFastTreeRegressionFeaturizationInPipeline(int iteration)
         {
+            Console.WriteLine(iteration);
             int dataPointCount = 200;
             var data = SamplesUtils.DatasetUtils.GenerateFloatLabelFloatFeatureVectorSamples(dataPointCount).ToList();
             var dataView = ML.Data.LoadFromEnumerable(data);