diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index ba92e36cbde..d0c60adf0be 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -718,10 +718,8 @@ stages:
           steps:
           - checkout: self
             clean: true
-          - script: eng\CIBuild.cmd -testBenchmarks
+          - script: eng\CIBuild.cmd -configuration $(_BuildConfig) -testBenchmarks
             displayName: Smoke test fast benchmarks
-            continueOnError: true
-            condition: always()
 
         # Test trimming on Windows
         - job: Build_And_Test_Trimming_Windows
diff --git a/eng/Build.ps1 b/eng/Build.ps1
index a971d399788..fe7eb56a5b1 100644
--- a/eng/Build.ps1
+++ b/eng/Build.ps1
@@ -346,7 +346,7 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
 
     if ($env:RunningAsPullRequest -ne "true" -and $noTestFilter -eq $false) {
         $args += " --filter TestCategory!=PullRequest"
-    }`
+    }
 
     if ($asBackgroundJob) {
         Write-Host("Starting on the background: $args")
@@ -549,11 +549,7 @@ try {
     }
 
     if ($testBenchmarks) {
-        $properties_storage = $properties
-        $properties += "/p:RuntimeIdentifier=win-x64"
-        $properties += "/p:Configuration=Release" # Always run in release.
         BuildSolution "FSharp.Benchmarks.sln" $False
-        $properties = $properties_storage
     }
 
     if ($pack) {
diff --git a/tests/benchmarks/FCSBenchmarks/BenchmarkComparison/HistoricalBenchmark.Runner/HistoricalBenchmark.Runner.fsproj b/tests/benchmarks/FCSBenchmarks/BenchmarkComparison/HistoricalBenchmark.Runner/HistoricalBenchmark.Runner.fsproj
index 13add70323f..fe6d95f9050 100644
--- a/tests/benchmarks/FCSBenchmarks/BenchmarkComparison/HistoricalBenchmark.Runner/HistoricalBenchmark.Runner.fsproj
+++ b/tests/benchmarks/FCSBenchmarks/BenchmarkComparison/HistoricalBenchmark.Runner/HistoricalBenchmark.Runner.fsproj
@@ -4,6 +4,7 @@
         <TargetFramework>net8.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <RootNamespace>HistoricalBenchmark.Utilities</RootNamespace>
+        <NoWarn>$(NoWarn);NETSDK1206</NoWarn>
     </PropertyGroup>
 
     <ItemGroup>
diff --git a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/BackgroundCompilerBenchmarks.fs b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/BackgroundCompilerBenchmarks.fs
index 528557fa3d7..b7ed2a4b78d 100644
--- a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/BackgroundCompilerBenchmarks.fs
+++ b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/BackgroundCompilerBenchmarks.fs
@@ -225,7 +225,7 @@ type TestProjectType =
 [<MemoryDiagnoser>]
 [<ThreadingDiagnoser>]
 [<SimpleJob(warmupCount=1,iterationCount=4)>]
-[<BenchmarkCategory(ShortCategory)>]
+[<BenchmarkCategory(LongCategory)>]
 type TransparentCompilerBenchmark() =
 
     let size = 30