fix(shim): Remove console window for GUI apps (#5559)

* Change shim subsystem to prevent console window for GUI apps.

* Removed redundant log

* Update core.ps1

* Fixes file access rights and the log message

* update changelog

* Update CHANGELOG.md (PR Number)

Co-authored-by: Rashil Gandhi <46838874+rashil2000@users.noreply.github.com>

---------

Co-authored-by: Quasar <suhas.nandibhatla@gmail.com>
Co-authored-by: Rashil Gandhi <46838874+rashil2000@users.noreply.github.com>

CHANGELOG.md

@@ -10,6 +10,7 @@
 
 ### Bug Fixes
 
+- **shim:** Remove console window for GUI applications ([#5559](https://github.com/ScoopInstaller/Scoop/issues/5559))
 - **decompress:** Exclude '*.nsis' that may cause error ([#5294](https://github.com/ScoopInstaller/Scoop/issues/5294))
 - **autoupdate:** Fix file hash extraction ([#5295](https://github.com/ScoopInstaller/Scoop/issues/5295))
 - **getopt:** Stop split arguments in `getopt()` and ensure array by explicit arguments type ([#5326](https://github.com/ScoopInstaller/Scoop/issues/5326))

lib/core.ps1

@@ -1,3 +1,56 @@
+# Returns the subsystem of the EXE
+function Get-Subsystem($filePath) {
+    try {
+        $fileStream = [System.IO.FileStream]::new($filePath, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read)
+        $binaryReader = [System.IO.BinaryReader]::new($fileStream)
+    } catch {
+        return -1  # leave the subsystem part silently
+    }
+
+    try {
+        $fileStream.Seek(0x3C, [System.IO.SeekOrigin]::Begin) | Out-Null
+        $peOffset = $binaryReader.ReadInt32()
+
+        $fileStream.Seek($peOffset, [System.IO.SeekOrigin]::Begin) | Out-Null
+        $fileHeaderOffset = $fileStream.Position
+
+        $fileStream.Seek(18, [System.IO.SeekOrigin]::Current) | Out-Null
+        $fileStream.Seek($fileHeaderOffset + 0x5C, [System.IO.SeekOrigin]::Begin) | Out-Null
+
+        return $binaryReader.ReadInt16()
+    } finally {
+        $binaryReader.Close()
+        $fileStream.Close()
+    }
+}
+
+function Change-Subsystem($filePath, $targetSubsystem) {
+    try {
+        $fileStream = [System.IO.FileStream]::new($filePath, [System.IO.FileMode]::Open, [System.IO.FileAccess]::ReadWrite)
+        $binaryReader = [System.IO.BinaryReader]::new($fileStream)
+        $binaryWriter = [System.IO.BinaryWriter]::new($fileStream)
+    } catch {
+        Write-Output "Error opening File:'$filePath'"
+        return
+    }
+
+    try {
+        $fileStream.Seek(0x3C, [System.IO.SeekOrigin]::Begin) | Out-Null
+        $peOffset = $binaryReader.ReadInt32()
+
+        $fileStream.Seek($peOffset, [System.IO.SeekOrigin]::Begin) | Out-Null
+        $fileHeaderOffset = $fileStream.Position
+
+        $fileStream.Seek(18, [System.IO.SeekOrigin]::Current) | Out-Null
+        $fileStream.Seek($fileHeaderOffset + 0x5C, [System.IO.SeekOrigin]::Begin) | Out-Null
+
+        $binaryWriter.Write([System.Int16] $targetSubsystem)
+    } finally {
+        $binaryReader.Close()
+        $fileStream.Close()
+    }
+}
+
 function Optimize-SecurityProtocol {
     # .NET Framework 4.7+ has a default security protocol called 'SystemDefault',
     # which allows the operating system to choose the best protocol to use.
@@ -839,6 +892,13 @@ function shim($path, $global, $name, $arg) {
         if ($arg) {
             Write-Output "args = $arg" | Out-UTF8File "$shim.shim" -Append
         }
+
+        $target_subsystem = Get-Subsystem $resolved_path
+
+        if (($target_subsystem -ne 3) -and ($target_subsystem -ge 0)) { # Subsystem -eq 3 means `Console`, -ge 0 to ignore
+            Write-Output "Making $shim.exe a GUI binary."
+            Change-Subsystem "$shim.exe" $target_subsystem
+        }
     } elseif ($path -match '\.(bat|cmd)$') {
         # shim .bat, .cmd so they can be used by programs with no awareness of PSH
         warn_on_overwrite "$shim.cmd" $path