diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml
index 0b4a9c655..1618e5cb3 100644
--- a/.github/workflows/bencher.yml
+++ b/.github/workflows/bencher.yml
@@ -67,6 +67,8 @@ jobs:
             source: script-cloud
           - os: ubuntu-latest
             source: script-gen
+          - os: ubuntu-latest
+            source: script-gen-version
           - os: ubuntu-latest
             source: cargo-main
           - os: ubuntu-latest
@@ -80,6 +82,8 @@ jobs:
             source: script-cloud
           - os: macos-latest
             source: script-gen
+          - os: macos-latest
+            source: script-gen-version
           - os: macos-latest
             source: cargo-main
           - os: macos-latest
@@ -93,6 +97,8 @@ jobs:
             source: script-cloud
           - os: windows-latest
             source: script-gen
+          - os: windows-latest
+            source: script-gen-version
           - os: windows-latest
             source: cargo-main
           - os: windows-latest
@@ -112,13 +118,19 @@ jobs:
         if: matrix.source == 'script-cloud' && matrix.os == 'windows-latest'
         run: irm https://bencher.dev/download/install-cli.ps1 | iex
       - uses: actions/checkout@v4
-        if: matrix.source == 'script-gen'
+        if: matrix.source == 'script-gen' || matrix.source == 'script-gen-version'
       - name: Unix CLI Install Script
         if: matrix.source == 'script-gen' && matrix.os != 'windows-latest'
         run: cat services/cli/templates/output/install-cli.sh | sh
+      - name: Unix CLI Install Script Version
+        if: matrix.source == 'script-gen-version' && matrix.os != 'windows-latest'
+        run: export BENCHER_VERSION=0.4.27; cat services/cli/templates/output/install-cli.sh | sh
       - name: Windows CLI Install Script
         if: matrix.source == 'script-gen' && matrix.os == 'windows-latest'
         run: gc -Raw services/cli/templates/output/install-cli.ps1 | iex
+      - name: Windows CLI Install Script Version
+        if: matrix.source == 'script-gen-version' && matrix.os == 'windows-latest'
+        run: $env:BENCHER_VERSION="0.4.27"; gc -Raw services/cli/templates/output/install-cli.ps1 | iex
       - name: Cargo CLI Install main
         if: matrix.source == 'cargo-main'
         run: cargo install --git https://github.com/bencherdev/bencher --branch main --locked bencher_cli
diff --git a/services/cli/templates/install-cli.ps1.j2 b/services/cli/templates/install-cli.ps1.j2
index dc5ab3d09..5bb2a1db0 100644
--- a/services/cli/templates/install-cli.ps1.j2
+++ b/services/cli/templates/install-cli.ps1.j2
@@ -20,6 +20,9 @@ If you get an error that says "running scripts is disabled on this system":
 - Enter: `Y`
 - Rerun this script
 
+.PARAMETER AppVersion
+The version of the application to install
+
 .PARAMETER ArtifactDownloadUrl
 The URL of the directory where artifacts can be fetched from
 
@@ -31,18 +34,18 @@ Print help
 
 #>
 
-$app_name = 'bencher'
-$app_version = if ($env:BENCHER_VERSION) { $env:BENCHER_VERSION } else { '{{ app_version }}' }
-
 param (
-    [Parameter(HelpMessage = "The URL of the directory where artifacts can be fetched from")]
-    [string]$ArtifactDownloadUrl = "https://bencher.dev/download/$app_version",
-    [Parameter(HelpMessage = "Don't add the install directory to PATH")]
-    [switch]$NoModifyPath,
-    [Parameter(HelpMessage = "Print Help")]
-    [switch]$Help
+  [Parameter(HelpMessage = "The version of the application to install")]
+  [string]$AppVersion = $(if ($env:BENCHER_VERSION) { $env:BENCHER_VERSION } else { '{{ app_version }}' }),
+  [Parameter(HelpMessage = "The URL of the directory where artifacts can be fetched from")]
+  [string]$ArtifactDownloadUrl = "https://bencher.dev/download/$(if ($env:BENCHER_VERSION) { $env:BENCHER_VERSION } else { '{{ app_version }}' })",
+  [Parameter(HelpMessage = "Don't add the install directory to PATH")]
+  [switch]$NoModifyPath,
+  [Parameter(HelpMessage = "Print Help")]
+  [switch]$Help
 )
 
+$app_name = 'bencher'
 
 function Install-Binary($install_args) {
   if ($Help) {
@@ -55,7 +58,7 @@ function Install-Binary($install_args) {
   $platforms = @{
   {%- for artifact in artifacts %}
     "{{ artifact.target_triple }}" = @{
-      "artifact_name" = "$app_name-v$app_version-{{ artifact.os_arch }}.exe"
+      "artifact_name" = "$app_name-v$AppVersion-{{ artifact.os_arch }}.exe"
       "zip_ext" = "{{ artifact.zip_style }}"
       "bins" = {% for bin in artifact.binaries -%}
         "{{ bin }}"{{ ", " if not loop.last else "" }}
@@ -137,7 +140,7 @@ https://github.com/bencherdev/bencher/issues
 
   # Download and unpack!
   $url = "$download_url/$artifact_name"
-  Write-Information "Downloading Bencher CLI ($app_name $app_version)"
+  Write-Information "Downloading Bencher CLI ($app_name $AppVersion)"
   Write-Verbose "  From: $url"
   Write-Verbose "  To:   $dir_path"
   $wc = New-Object Net.Webclient
diff --git a/services/cli/templates/output/install-cli.ps1 b/services/cli/templates/output/install-cli.ps1
index c169048f8..9486a4955 100755
--- a/services/cli/templates/output/install-cli.ps1
+++ b/services/cli/templates/output/install-cli.ps1
@@ -20,6 +20,9 @@ If you get an error that says "running scripts is disabled on this system":
 - Enter: `Y`
 - Rerun this script
 
+.PARAMETER AppVersion
+The version of the application to install
+
 .PARAMETER ArtifactDownloadUrl
 The URL of the directory where artifacts can be fetched from
 
@@ -31,18 +34,18 @@ Print help
 
 #>
 
-$app_name = 'bencher'
-$app_version = if ($env:BENCHER_VERSION) { $env:BENCHER_VERSION } else { '0.4.28' }
-
 param (
-    [Parameter(HelpMessage = "The URL of the directory where artifacts can be fetched from")]
-    [string]$ArtifactDownloadUrl = "https://bencher.dev/download/$app_version",
-    [Parameter(HelpMessage = "Don't add the install directory to PATH")]
-    [switch]$NoModifyPath,
-    [Parameter(HelpMessage = "Print Help")]
-    [switch]$Help
+  [Parameter(HelpMessage = "The version of the application to install")]
+  [string]$AppVersion = $(if ($env:BENCHER_VERSION) { $env:BENCHER_VERSION } else { '0.4.28' }),
+  [Parameter(HelpMessage = "The URL of the directory where artifacts can be fetched from")]
+  [string]$ArtifactDownloadUrl = "https://bencher.dev/download/$(if ($env:BENCHER_VERSION) { $env:BENCHER_VERSION } else { '0.4.28' })",
+  [Parameter(HelpMessage = "Don't add the install directory to PATH")]
+  [switch]$NoModifyPath,
+  [Parameter(HelpMessage = "Print Help")]
+  [switch]$Help
 )
 
+$app_name = 'bencher'
 
 function Install-Binary($install_args) {
   if ($Help) {
@@ -54,13 +57,13 @@ function Install-Binary($install_args) {
 
   $platforms = @{
     "x86_64-pc-windows-msvc" = @{
-      "artifact_name" = "$app_name-v$app_version-windows-x86-64.exe"
+      "artifact_name" = "$app_name-v$AppVersion-windows-x86-64.exe"
       "zip_ext" = ""
       "bins" = "bencher"
       "bin" = "bencher"
     }
     "aarch64-pc-windows-msvc" = @{
-      "artifact_name" = "$app_name-v$app_version-windows-arm-64.exe"
+      "artifact_name" = "$app_name-v$AppVersion-windows-arm-64.exe"
       "zip_ext" = ""
       "bins" = "bencher"
       "bin" = "bencher"
@@ -139,7 +142,7 @@ https://github.com/bencherdev/bencher/issues
 
   # Download and unpack!
   $url = "$download_url/$artifact_name"
-  Write-Information "Downloading Bencher CLI ($app_name $app_version)"
+  Write-Information "Downloading Bencher CLI ($app_name $AppVersion)"
   Write-Verbose "  From: $url"
   Write-Verbose "  To:   $dir_path"
   $wc = New-Object Net.Webclient