feat(core): Add 'Get-Encoding()' function to fix missing webClient encoding

CHANGELOG.md

@@ -1,3 +1,9 @@
+## [Unreleased](https://github.com/ScoopInstaller/Scoop/compare/master...develop)
+
+### Features
+
+- **core:** Add `Get-Encoding` function to fix missing webclient encoding ([#4956](https://github.com/ScoopInstaller/Scoop/issues/4956))
+
 ## [v0.2.1](https://github.com/ScoopInstaller/Scoop/compare/v0.2.0...v0.2.1) - 2022-06-10
 
 ### Features

bin/checkver.ps1

@@ -113,7 +113,7 @@ $Queue | ForEach-Object {
     } else {
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
     }
-    Register-ObjectEvent $wc downloadstringcompleted -ErrorAction Stop | Out-Null
+    Register-ObjectEvent $wc downloadDataCompleted -ErrorAction Stop | Out-Null
 
     $githubRegex = '\/releases\/tag\/(?:v|V)?([\d.]+)'
 
@@ -190,7 +190,7 @@ $Queue | ForEach-Object {
     }
 
     $wc.Headers.Add('Referer', (strip_filename $url))
-    $wc.DownloadStringAsync($url, $state)
+    $wc.DownloadDataAsync($url, $state)
 }
 
 function next($er) {
@@ -218,7 +218,7 @@ while ($in_progress -gt 0) {
     $ver = $Version
 
     if (!$ver) {
-        $page = $ev.SourceEventArgs.Result
+        $page = (Get-Encoding($wc)).GetString($ev.SourceEventArgs.Result)
         $err = $ev.SourceEventArgs.Error
         if ($json.checkver.script) {
             $page = Invoke-Command ([scriptblock]::Create($json.checkver.script -join "`r`n"))

bin/describe.ps1

@@ -44,7 +44,8 @@ $Queue | ForEach-Object {
     try {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $home_html = $wc.DownloadString($manifest.homepage)
+        $homepage = $wc.DownloadData($manifest.homepage)
+        $home_html = (Get-Encoding($wc)).GetString($homepage)
     } catch {
         Write-Host "`n$($_.Exception.Message)" -ForegroundColor Red
         return

lib/autoupdate.ps1

@@ -1,20 +1,21 @@
 # Must included with 'json.ps1'
 function find_hash_in_rdf([String] $url, [String] $basename) {
-    $data = $null
+    $xml = $null
     try {
         # Download and parse RDF XML file
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        [xml]$data = $wc.downloadstring($url)
+        $data = $wc.DownloadData($url)
+        [xml]$xml = (Get-Encoding($wc)).GetString($data)
     } catch [system.net.webexception] {
         write-host -f darkred $_
         write-host -f darkred "URL $url is not valid"
         return $null
     }
 
     # Find file content
-    $digest = $data.RDF.Content | Where-Object { [String]$_.about -eq $basename }
+    $digest = $xml.RDF.Content | Where-Object { [String]$_.about -eq $basename }
 
     return format_hash $digest.sha256
 }
@@ -35,7 +36,8 @@ function find_hash_in_textfile([String] $url, [Hashtable] $substitutions, [Strin
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $hashfile = $wc.downloadstring($url)
+        $data = $wc.DownloadData($url)
+        $hashfile = (Get-Encoding($wc)).GetString($data)
     } catch [system.net.webexception] {
         write-host -f darkred $_
         write-host -f darkred "URL $url is not valid"
@@ -88,7 +90,8 @@ function find_hash_in_json([String] $url, [Hashtable] $substitutions, [String] $
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $json = $wc.downloadstring($url)
+        $data = $wc.DownloadData($url)
+        $json = (Get-Encoding($wc)).GetString($data)
     } catch [system.net.webexception] {
         write-host -f darkred $_
         write-host -f darkred "URL $url is not valid"
@@ -108,7 +111,8 @@ function find_hash_in_xml([String] $url, [Hashtable] $substitutions, [String] $x
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $xml = [xml]$wc.downloadstring($url)
+        $data = $wc.DownloadData($url)
+        $xml = [xml]((Get-Encoding($wc)).GetString($data))
     } catch [system.net.webexception] {
         write-host -f darkred $_
         write-host -f darkred "URL $url is not valid"

lib/core.ps1

@@ -15,6 +15,14 @@ function Optimize-SecurityProtocol {
     }
 }
 
+function Get-Encoding($wc) {
+    if ($null -ne $wc.ResponseHeaders -and $wc.ResponseHeaders['Content-Type'] -match 'charset=([^;]*)') {
+        return [System.Text.Encoding]::GetEncoding($Matches[1])
+    } else {
+        return [System.Text.Encoding]::GetEncoding('utf-8')
+    }
+}
+
 function Get-UserAgent() {
     return "Scoop/1.0 (+http://scoop.sh/) PowerShell/$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor) (Windows NT $([System.Environment]::OSVersion.Version.Major).$([System.Environment]::OSVersion.Version.Minor); $(if($env:PROCESSOR_ARCHITECTURE -eq 'AMD64'){'Win64; x64; '})$(if($env:PROCESSOR_ARCHITEW6432 -eq 'AMD64'){'WOW64; '})$PSEdition)"
 }

lib/description.ps1

@@ -18,7 +18,8 @@ function find_description($url, $html, $redir = $false) {
     if($refresh -and !$redir) {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $html = $wc.downloadstring($refresh)
+        $data = $wc.DownloadData($refresh)
+        $html = (Get-Encoding($wc)).GetString($data)
         return find_description $refresh $html $true
     }
 

lib/manifest.ps1

@@ -12,7 +12,8 @@ function url_manifest($url) {
     try {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $str = $wc.downloadstring($url)
+        $data = $wc.DownloadData($url)
+        $str = (Get-Encoding($wc)).GetString($data)
     } catch [system.management.automation.methodinvocationexception] {
         warn "error: $($_.exception.innerexception.message)"
     } catch {
@@ -66,7 +67,8 @@ function save_installed_manifest($app, $bucket, $dir, $url) {
     if ($url) {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $wc.downloadstring($url) > "$dir\manifest.json"
+        $data = $wc.DownloadData($url)
+        (Get-Encoding($wc)).GetString($data) | Out-UTF8File "$dir\manifest.json"
     } else {
         Copy-Item (manifest_path $app $bucket) "$dir\manifest.json"
     }

libexec/scoop-virustotal.ps1

@@ -85,7 +85,8 @@ Function Get-VirusTotalResult($hash, $app) {
     $url = "https://www.virustotal.com/ui/files/$hash"
     $wc = New-Object Net.Webclient
     $wc.Headers.Add('User-Agent', (Get-UserAgent))
-    $result = $wc.downloadstring($url)
+    $data = $wc.DownloadData($url)
+    $result = (Get-Encoding($wc)).GetString($data)
     $stats = json_path $result '$.data.attributes.last_analysis_stats'
     $malicious = json_path $stats '$.malicious'
     $suspicious = json_path $stats '$.suspicious'