From 37f448b99feedc6bb863a00925a94cc8d73ba713 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 00:43:38 +0800 Subject: [PATCH 01/28] feat(scoop-virustotal): migrate to virustotal api v3, improve flows, and use standard psstreams --- libexec/scoop-virustotal.ps1 | 285 +++++++++++++++++++++++------------ 1 file changed, 187 insertions(+), 98 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 5be7a10f56..fdafae470b 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -1,28 +1,21 @@ # Usage: scoop virustotal [* | app1 app2 ...] [options] -# Summary: Look for app's hash on virustotal.com -# Help: Look for app's hash (MD5, SHA1 or SHA256) on virustotal.com +# Summary: Look for app's hash or url on virustotal.com +# Help: Look for app's hash or url on virustotal.com # # Use a single '*' for app to check all installed apps. # -# The download's hash is also a key to access VirusTotal's scan results. -# This allows to check the safety of the files without even downloading -# them in many cases. If the hash is unknown to VirusTotal, the -# download link is printed to submit it to VirusTotal. -# -# If you have signed up to VirusTotal's community, you have an API key -# that this script can use to submit unknown packages for inspection -# if you use the `--scan' flag. Tell scoop about your API key with: +# To use this command, you have to sign up to VirusTotal's community, +# and get an API key. Then, tell scoop about your API key with: # # scoop config virustotal_api_key # # Exit codes: -# 0 -> success -# 1 -> problem parsing arguments -# 2 -> at least one package was marked unsafe by VirusTotal -# 4 -> at least one exception was raised while looking for info -# 8 -> at least one package couldn't be queried because its hash type -# isn't supported by VirusTotal, the manifest couldn't be found -# or didn't contain a hash +# 0 -> success +# 1 -> problem parsing arguments +# 2 -> at least one package was marked unsafe by VirusTotal +# 4 -> at least one exception was raised while looking for info +# 8 -> at least one package couldn't be queried because the manifest couldn't be found +# 16 -> VirusTotal API key is not configured # Note: the exit codes (2, 4 & 8) may be combined, e.g. 6 -> exit codes # 2 & 4 combined # @@ -67,11 +60,17 @@ if (!$opt.n -and !$opt.'no-depends') { $_ERR_UNSAFE = 2 $_ERR_EXCEPTION = 4 $_ERR_NO_INFO = 8 +$_ERR_NO_API_KEY = 16 $exit_code = 0 -# Global flag to warn only once about missing API key: -$warned_no_api_key = $False +# Global API key: +$api_key = get_config virustotal_api_key +if (!$api_key) { + Write-Warning ("VirusTotal API key is not configured`n`n" + + "`tscoop config virustotal_api_key ") + exit $_ERR_NO_API_KEY +} # Global flag to explain only once about sleep between requests $explained_rate_limit_sleeping = $False @@ -80,65 +79,124 @@ $explained_rate_limit_sleeping = $False # script execution progresses $requests = 0 -Function Get-VirusTotalResult($hash, $app) { +Function ConvertTo-VirusTotalUrlId ($url) { + $url_id = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($url)) + $url_id = $url_id -replace '\+', '-' + $url_id = $url_id -replace '/', '_' + $url_id = $url_id -replace '=', '' + $url_id +} + +Function Get-RemoteFileSize ($url) { + $response = Invoke-WebRequest -Uri $url -Method HEAD -UseBasicParsing + $response.Headers.'Content-Length' | ForEach-Object { + ([System.Convert]::ToDouble($_) / 1048576) + } +} + +Function Get-VirusTotalResultByHash ($hash, $app) { $hash = $hash.ToLower() - $url = "https://www.virustotal.com/ui/files/$hash" - $wc = New-Object Net.Webclient - $wc.Headers.Add('User-Agent', (Get-UserAgent)) - $result = $wc.downloadstring($url) + $api_url = "https://www.virustotal.com/api/v3/files/$hash" + $headers = @{} + $headers.Add('Accept', 'application/json') + $headers.Add('x-apikey', $api_key) + $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing + $result = $response.Content $stats = json_path $result '$.data.attributes.last_analysis_stats' - $malicious = json_path $stats '$.malicious' - $suspicious = json_path $stats '$.suspicious' - $undetected = json_path $stats '$.undetected' - $unsafe = [int]$malicious + [int]$suspicious - $see_url = "see https://www.virustotal.com/#/file/$hash/detection" - switch ($unsafe) { - 0 { if ($undetected -eq 0) { $fg = 'Yellow' } else { $fg = 'DarkGreen' } } - 1 { $fg = 'DarkYellow' } - 2 { $fg = 'Yellow' } - default { $fg = 'Red' } + [int]$malicious = json_path $stats '$.malicious' + [int]$suspicious = json_path $stats '$.suspicious' + [int]$timeout = json_path $stats '$.timeout' + [int]$undetected = json_path $stats '$.undetected' + [int]$unsafe = $malicious + $suspicious + [int]$total = $unsafe + $undetected + $report_url = "https://www.virustotal.com/gui/file/$hash" + if ($undetected -eq 0) { + Write-Information "INFO : $app`: Analysis is in progress." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Hash' = $hash + FileReport = $report_url + } + } else { + switch ($unsafe) { + 0 { + Write-Information "$($PSStyle.Foreground.BrightGreen)INFO : $app`: $unsafe / $total$($PSStyle.Reset)" -InformationAction 'Continue' + } + 1 { + Write-Warning "$app`: $unsafe / $total" + } + 2 { + Write-Warning "$app`: $unsafe / $total" + } + Default { + Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning) / $total" + } + } + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Hash' = $hash + Malicious = $malicious + Suspicious = $suspicious + Timeout = $timeout + Undetected = $undetected + FileReport = $report_url + } } - Write-Host -f $fg "$app`: $unsafe/$undetected, $see_url" if ($unsafe -gt 0) { - return $_ERR_UNSAFE + $Script:exit_code = $exit_code -bor $_ERR_UNSAFE } - return 0 } -Function Search-VirusTotal ($hash, $app) { - if ($hash -match '(?[^:]+):(?.*)') { - $hash = $matches['hash'] - if ($matches['algo'] -match '(md5|sha1|sha256)') { - return Get-VirusTotalResult $hash $app - } else { - warn "$app`: Unsupported hash $($matches['algo']). VirusTotal needs md5, sha1 or sha256." - return $_ERR_NO_INFO +Function Get-VirusTotalResultByUrl ($url, $app) { + $id = ConvertTo-VirusTotalUrlId $url + $api_url = "https://www.virustotal.com/api/v3/urls/$id" + $headers = @{} + $headers.Add('Accept', 'application/json') + $headers.Add('x-apikey', $api_key) + $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing + $result = $response.Content + $id = json_path $result '$.data.id' + $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null + $url_report_url = "https://www.virustotal.com/gui/url/$id" + Write-Information "INFO : $app`: Url report found: $url_report_url" -InformationAction 'Continue' + Write-Information "INFO : $($PSStyle.Formatting.Warning)Url report is not accurate.$($PSStyle.Reset)" -InformationAction 'Continue' + if (!$hash) { + Write-Warning "$app`: File report not found. Manual file upload is required (instead of url submission)." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + UrlReport = $url_report_url + } + } else { + Write-Information "INFO : $app`: Related file report found." -InformationAction 'Continue' + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Hash' = $hash + 'App.Url' = $url + UrlReport = $url_report_url } } - - return Get-VirusTotalResult $hash $app } -Function Submit-RedirectedUrl { - # Follow up to one level of HTTP redirection - # - # Copied from http://www.powershellmagazine.com/2013/01/29/pstip-retrieve-a-redirected-url/ - # Adapted according to Roy's response (January 23, 2014 at 11:59 am) - # Adapted to always return an URL - Param ( - [Parameter(Mandatory = $true)] - [String]$URL - ) - $request = [System.Net.WebRequest]::Create($url) - $request.AllowAutoRedirect = $false - $response = $request.GetResponse() - if (([int]$response.StatusCode -ge 300) -and ([int]$response.StatusCode -lt 400)) { - $redir = $response.GetResponseHeader('Location') +Function Get-VirusTotalResult ($hash, $url, $app) { + if ($hash -match '(?[^:]+):(?.*)') { + $hash = $matches.hash + if ($matches.algo -match '(md5|sha1|sha256)') { + Get-VirusTotalResultByHash $hash $app + return + } + Write-Warning "$app`: Unsupported hash $($matches.algo), will search by url instead" + } elseif ($hash) { + Get-VirusTotalResultByHash $hash $app + return } else { - $redir = $URL + Write-Warning "$app`: Can't find hash for $url, will search by url instead" + } + $result = Get-VirusTotalResultByUrl $url $app + $hash = $result.'App.Hash' + if ($hash) { + Get-VirusTotalResultByHash $result.'App.Hash' $app } - $response.Close() - return $redir } # Submit-ToVirusTotal @@ -152,32 +210,32 @@ Function Submit-RedirectedUrl { # exceeded, without risking an infinite loop (as stack # overflow) if the submission keeps failing. Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { - $api_key = get_config virustotal_api_key - if ($do_scan -and !$api_key -and !$warned_no_api_key) { - $warned_no_api_key = $true - info 'Submitting unknown apps needs a VirusTotal API key. ' + - "Set it up with`n`tscoop config virustotal_api_key " - - } - if (!$do_scan -or !$api_key) { - warn "$app`: not found`: manually submit $url" + if (!$do_scan) { + warn "$app`: not found`: you can manually submit $url" return } try { - # Follow redirections (for e.g. sourceforge URLs) because - # VirusTotal analyzes only "direct" download links - $url = $url.Split('#').GetValue(0) - $new_redir = $url - do { - $orig_redir = $new_redir - $new_redir = Submit-RedirectedUrl $orig_redir - } while ($orig_redir -ne $new_redir) $requests += 1 - $result = Invoke-WebRequest -Uri 'https://www.virustotal.com/vtapi/v2/url/scan' -Body @{apikey = $api_key; url = $new_redir } -Method Post -UseBasicParsing - $submitted = $result.StatusCode -eq 200 - if ($submitted) { - warn "$app`: not found`: submitted $url" + + $encoded_url = [System.Web.HttpUtility]::UrlEncode($url) + $api_url = 'https://www.virustotal.com/api/v3/urls' + $content_type = 'application/x-www-form-urlencoded' + $headers = @{} + $headers.Add('Accept', 'application/json') + $headers.Add('x-apikey', $api_key) + $headers.Add('Content-Type', $content_type) + $body = "url=$encoded_url" + $result = Invoke-WebRequest -Uri $api_url -Method POST -Headers $headers -ContentType $content_type -Body $body -UseBasicParsing + if ($result.StatusCode -eq 200) { + $id = ((json_path $result '$.data.id') -split '-')[1] + $url_report_url = "https://www.virustotal.com/gui/url/$id" + $fileSize = Get-RemoteFileSize $url + Write-Information "INFO : $app`: Remote file size: $($fileSize.ToString('0.00 MB'))" -InformationAction 'Continue' + if ($fileSize -gt 70) { + Write-Information "INFO : $($PSStyle.Formatting.Warning)Large files might require manual file upload instead of url submission.$($PSStyle.Reset)" + } + Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' return } @@ -185,26 +243,26 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { if (!$retrying) { if (!$explained_rate_limit_sleeping) { $explained_rate_limit_sleeping = $True - info "Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" + Write-Information "INFO : Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" -InformationAction 'Continue' } Start-Sleep -s (60 + $requests) - Submit-ToVirusTotal $new_redir $app $do_scan $True + Submit-ToVirusTotal $url $app $do_scan $True } else { - warn "$app`: VirusTotal submission of $url failed`:`n" + + Write-Warning "$app`: VirusTotal submission of $url failed`:`n" + "`tAPI returned $($result.StatusCode) after retrying" } } catch [Exception] { - warn "$app`: VirusTotal submission failed`: $($_.Exception.Message)" + Write-Warning "$app`: VirusTotal submission failed`: $($_.Exception.Message)" return } } -foreach ($app in $apps) { - # write-host $app +$apps | ForEach-Object { + $app = $_ $null, $manifest, $bucket, $null = Get-Manifest $app if (!$manifest) { $exit_code = $exit_code -bor $_ERR_NO_INFO - warn "$app`: manifest not found" + Write-Warning "$app`: manifest not found" continue } @@ -214,20 +272,51 @@ foreach ($app in $apps) { $hash = hash_for_url $manifest $url $architecture try { + if ($hash -match '(?[^:]+):(?.*)') { + $hash = $matches.hash + if ($matches.algo -notmatch '(md5|sha1|sha256)') { + $hash = $null + Write-Warning "$app`: Unsupported hash $($matches.algo). Will search by url instead." + } + } if ($hash) { - $exit_code = $exit_code -bor (Search-VirusTotal $hash $app) + $report = Get-VirusTotalResultByHash $hash $app + $report + return } else { - warn "$app`: Can't find hash for $url" + Write-Warning "$app`: Hash for $url not found. Will search by url instead." + } + } catch [Exception] { + $exit_code = $exit_code -bor $_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + Write-Warning "$app`: File report not found. Will search by url instead." + } else { + if ($_.Exception.Response.StatusCode -in 204, 429) { + abort "$app`: VirusTotal request failed`: $($_.Exception.Message)", $exit_code + } + Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" + return + } + } + + try { + $url_report = Get-VirusTotalResultByUrl $url $app + $hash = $url_report.'App.Hash' + if ($hash) { + $report = Get-VirusTotalResultByHash $hash $app + $report } } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION - if ($_.Exception.Message -like '*(404)*') { + if ($_.Exception.Response.StatusCode -eq 404) { + Write-Warning "$app`: Url report not found. Will submit $url" Submit-ToVirusTotal $url $app ($opt.scan -or $opt.s) } else { - if ($_.Exception.Message -match '\(204|429\)') { + if ($_.Exception.Response.StatusCode -in 204, 429) { abort "$app`: VirusTotal request failed`: $($_.Exception.Message)", $exit_code } - warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" + Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" + return } } } From 94bacbc548c96147c4a20ea28aadc8c0582296fe Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 18:36:32 +0800 Subject: [PATCH 02/28] remove unused codes --- libexec/scoop-virustotal.ps1 | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index fdafae470b..55550f8215 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -178,27 +178,6 @@ Function Get-VirusTotalResultByUrl ($url, $app) { } } -Function Get-VirusTotalResult ($hash, $url, $app) { - if ($hash -match '(?[^:]+):(?.*)') { - $hash = $matches.hash - if ($matches.algo -match '(md5|sha1|sha256)') { - Get-VirusTotalResultByHash $hash $app - return - } - Write-Warning "$app`: Unsupported hash $($matches.algo), will search by url instead" - } elseif ($hash) { - Get-VirusTotalResultByHash $hash $app - return - } else { - Write-Warning "$app`: Can't find hash for $url, will search by url instead" - } - $result = Get-VirusTotalResultByUrl $url $app - $hash = $result.'App.Hash' - if ($hash) { - Get-VirusTotalResultByHash $result.'App.Hash' $app - } -} - # Submit-ToVirusTotal # - $url: where file to check can be downloaded # - $app: Name of the application (used for reporting) From bc6fa8cfca8807ba7d7d5ebe707ce7c937f227a1 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 18:39:38 +0800 Subject: [PATCH 03/28] fix bug --- libexec/scoop-virustotal.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 55550f8215..df010aff87 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -242,7 +242,7 @@ $apps | ForEach-Object { if (!$manifest) { $exit_code = $exit_code -bor $_ERR_NO_INFO Write-Warning "$app`: manifest not found" - continue + return } $urls = script:url $manifest $architecture From 8a497fadae0d6ebe7302bd8a591e94fbde576f68 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 18:47:50 +0800 Subject: [PATCH 04/28] adjust output --- libexec/scoop-virustotal.ps1 | 59 ++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index df010aff87..fc8942f887 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -116,30 +116,36 @@ Function Get-VirusTotalResultByHash ($hash, $app) { 'App.Name' = $app 'App.Hash' = $hash FileReport = $report_url + Malicious = $null + Suspicious = $null + Timeout = $null + Undetected = $null + UrlReport = $null } } else { switch ($unsafe) { 0 { - Write-Information "$($PSStyle.Foreground.BrightGreen)INFO : $app`: $unsafe / $total$($PSStyle.Reset)" -InformationAction 'Continue' + Write-Information "$($PSStyle.Foreground.BrightGreen)INFO : $app`: $unsafe/$total$($PSStyle.Reset)" -InformationAction 'Continue' } 1 { - Write-Warning "$app`: $unsafe / $total" + Write-Warning "$app`: $unsafe/$total" } 2 { - Write-Warning "$app`: $unsafe / $total" + Write-Warning "$app`: $unsafe $total" } Default { - Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning) / $total" + Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning)/$total" } } [PSCustomObject] @{ 'App.Name' = $app 'App.Hash' = $hash + FileReport = $report_url Malicious = $malicious Suspicious = $suspicious Timeout = $timeout Undetected = $undetected - FileReport = $report_url + UrlReport = $null } } if ($unsafe -gt 0) { @@ -158,21 +164,29 @@ Function Get-VirusTotalResultByUrl ($url, $app) { $id = json_path $result '$.data.id' $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null $url_report_url = "https://www.virustotal.com/gui/url/$id" - Write-Information "INFO : $app`: Url report found: $url_report_url" -InformationAction 'Continue' - Write-Information "INFO : $($PSStyle.Formatting.Warning)Url report is not accurate.$($PSStyle.Reset)" -InformationAction 'Continue' + Write-Information "INFO : $app`: Url report found." if (!$hash) { - Write-Warning "$app`: File report not found. Manual file upload is required (instead of url submission)." + Write-Information "INFO : $app`: Related file report not found." + Write-Warning "$app`: Manual file upload is required (instead of url submission)." [PSCustomObject] @{ 'App.Name' = $app - 'App.Url' = $url + FileReport = $null + Malicious = $null + Suspicious = $null + Timeout = $null + Undetected = $null UrlReport = $url_report_url } } else { - Write-Information "INFO : $app`: Related file report found." -InformationAction 'Continue' + Write-Information "INFO : $app`: Related file report found." [PSCustomObject] @{ 'App.Name' = $app 'App.Hash' = $hash - 'App.Url' = $url + FileReport = $null + Malicious = $null + Suspicious = $null + Timeout = $null + Undetected = $null UrlReport = $url_report_url } } @@ -210,9 +224,8 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { $id = ((json_path $result '$.data.id') -split '-')[1] $url_report_url = "https://www.virustotal.com/gui/url/$id" $fileSize = Get-RemoteFileSize $url - Write-Information "INFO : $app`: Remote file size: $($fileSize.ToString('0.00 MB'))" -InformationAction 'Continue' if ($fileSize -gt 70) { - Write-Information "INFO : $($PSStyle.Formatting.Warning)Large files might require manual file upload instead of url submission.$($PSStyle.Reset)" + Write-Information "INFO : $app`: Remote file size: $($fileSize.ToString('0.00 MB')). Large files might require manual file upload instead of url submission." -InformationAction 'Continue' } Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' return @@ -236,7 +249,7 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { } } -$apps | ForEach-Object { +$reports = $apps | ForEach-Object { $app = $_ $null, $manifest, $bucket, $null = Get-Manifest $app if (!$manifest) { @@ -251,19 +264,21 @@ $apps | ForEach-Object { $hash = hash_for_url $manifest $url $architecture try { + $isHashUnsupported = $false if ($hash -match '(?[^:]+):(?.*)') { $hash = $matches.hash if ($matches.algo -notmatch '(md5|sha1|sha256)') { $hash = $null + $isHashUnsupported = $true Write-Warning "$app`: Unsupported hash $($matches.algo). Will search by url instead." } } if ($hash) { - $report = Get-VirusTotalResultByHash $hash $app - $report + $file_report = Get-VirusTotalResultByHash $hash $app + $file_report return - } else { - Write-Warning "$app`: Hash for $url not found. Will search by url instead." + } elseif (!$isHashUnsupported) { + Write-Warning "$app`: Hash not found. Will search by url instead." } } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION @@ -282,8 +297,11 @@ $apps | ForEach-Object { $url_report = Get-VirusTotalResultByUrl $url $app $hash = $url_report.'App.Hash' if ($hash) { - $report = Get-VirusTotalResultByHash $hash $app - $report + $file_report = Get-VirusTotalResultByHash $hash $app + $file_report.'UrlReport' = $url_report.'UrlReport' + $file_report + } else { + $url_report } } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION @@ -300,5 +318,6 @@ $apps | ForEach-Object { } } } +$reports exit $exit_code From 5f5eb325d9d5066fe2d31934e8984ae89dcef4c4 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 19:09:37 +0800 Subject: [PATCH 05/28] fix --- libexec/scoop-virustotal.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index fc8942f887..b74efecdf8 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -110,7 +110,7 @@ Function Get-VirusTotalResultByHash ($hash, $app) { [int]$unsafe = $malicious + $suspicious [int]$total = $unsafe + $undetected $report_url = "https://www.virustotal.com/gui/file/$hash" - if ($undetected -eq 0) { + if ($total -eq 0) { Write-Information "INFO : $app`: Analysis is in progress." [PSCustomObject] @{ 'App.Name' = $app From 81e759e8433b2df8a2d722411a41e07edc47e048 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 19:25:41 +0800 Subject: [PATCH 06/28] fix output --- libexec/scoop-virustotal.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index b74efecdf8..6f77d60b74 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -131,7 +131,7 @@ Function Get-VirusTotalResultByHash ($hash, $app) { Write-Warning "$app`: $unsafe/$total" } 2 { - Write-Warning "$app`: $unsafe $total" + Write-Warning "$app`: $unsafe/$total" } Default { Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning)/$total" From 1a40167787e37fdee52d8e2a4449cadff69af00c Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sun, 12 Jun 2022 15:18:37 +0800 Subject: [PATCH 07/28] Adjust output --- libexec/scoop-virustotal.ps1 | 58 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 6f77d60b74..f747b91443 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -40,7 +40,7 @@ $architecture = ensure_architecture ($opt.a + $opt.arch) if (is_scoop_outdated) { if ($opt.u -or $opt.'no-update-scoop') { - warn 'Scoop is out of date.' + Write-Warning 'Scoop is out of date.' } else { scoop update } @@ -109,18 +109,16 @@ Function Get-VirusTotalResultByHash ($hash, $app) { [int]$undetected = json_path $stats '$.undetected' [int]$unsafe = $malicious + $suspicious [int]$total = $unsafe + $undetected + $fileSize = ([System.Convert]::ToDouble((json_path $result '$.data.attributes.size')) / 1048576) $report_url = "https://www.virustotal.com/gui/file/$hash" if ($total -eq 0) { - Write-Information "INFO : $app`: Analysis is in progress." + Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $hash - FileReport = $report_url - Malicious = $null - Suspicious = $null - Timeout = $null - Undetected = $null - UrlReport = $null + 'App.Name' = $app + 'App.Hash' = $hash + 'App.Size (MB)' = $fileSize.ToString('0.00') + FileReport = $report_url + UrlReport = $null } } else { switch ($unsafe) { @@ -138,14 +136,15 @@ Function Get-VirusTotalResultByHash ($hash, $app) { } } [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $hash - FileReport = $report_url - Malicious = $malicious - Suspicious = $suspicious - Timeout = $timeout - Undetected = $undetected - UrlReport = $null + 'App.Name' = $app + 'App.Hash' = $hash + 'App.Size (MB)' = $fileSize.ToString('0.00') + FileReport = $report_url + Malicious = $malicious + Suspicious = $suspicious + Timeout = $timeout + Undetected = $undetected + UrlReport = $null } } if ($unsafe -gt 0) { @@ -170,11 +169,8 @@ Function Get-VirusTotalResultByUrl ($url, $app) { Write-Warning "$app`: Manual file upload is required (instead of url submission)." [PSCustomObject] @{ 'App.Name' = $app + 'App.Hash' = $null FileReport = $null - Malicious = $null - Suspicious = $null - Timeout = $null - Undetected = $null UrlReport = $url_report_url } } else { @@ -183,10 +179,6 @@ Function Get-VirusTotalResultByUrl ($url, $app) { 'App.Name' = $app 'App.Hash' = $hash FileReport = $null - Malicious = $null - Suspicious = $null - Timeout = $null - Undetected = $null UrlReport = $url_report_url } } @@ -204,7 +196,7 @@ Function Get-VirusTotalResultByUrl ($url, $app) { # overflow) if the submission keeps failing. Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { if (!$do_scan) { - warn "$app`: not found`: you can manually submit $url" + Write-Warning "$app`: not found`: you can manually submit $url" return } @@ -228,6 +220,12 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { Write-Information "INFO : $app`: Remote file size: $($fileSize.ToString('0.00 MB')). Large files might require manual file upload instead of url submission." -InformationAction 'Continue' } Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Size (MB)' = $fileSize.ToString('0.00') + FileReport = $null + UrlReport = $url_report_url + } return } @@ -286,7 +284,8 @@ $reports = $apps | ForEach-Object { Write-Warning "$app`: File report not found. Will search by url instead." } else { if ($_.Exception.Response.StatusCode -in 204, 429) { - abort "$app`: VirusTotal request failed`: $($_.Exception.Message)", $exit_code + Write-Error "$app`: VirusTotal request failed`: $($_.Exception.Message)" + exit $exit_code } Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" return @@ -310,7 +309,8 @@ $reports = $apps | ForEach-Object { Submit-ToVirusTotal $url $app ($opt.scan -or $opt.s) } else { if ($_.Exception.Response.StatusCode -in 204, 429) { - abort "$app`: VirusTotal request failed`: $($_.Exception.Message)", $exit_code + Write-Error "$app`: VirusTotal request failed`: $($_.Exception.Message)" + exit $exit_code } Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" return From 8737f9eca1a289681ecf2ba4ca71b887acf1ac26 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sun, 12 Jun 2022 21:22:34 +0800 Subject: [PATCH 08/28] show vendors directly if malicious/suspicious != 0 --- libexec/scoop-virustotal.ps1 | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index f747b91443..be83fe7d81 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -121,6 +121,7 @@ Function Get-VirusTotalResultByHash ($hash, $app) { UrlReport = $null } } else { + $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value switch ($unsafe) { 0 { Write-Information "$($PSStyle.Foreground.BrightGreen)INFO : $app`: $unsafe/$total$($PSStyle.Reset)" -InformationAction 'Continue' @@ -135,13 +136,19 @@ Function Get-VirusTotalResultByHash ($hash, $app) { Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning)/$total" } } + $maliciousResults = $vendorResults | + Where-Object -Property category -EQ 'malicious' | + Select-Object -ExpandProperty engine_name + $suspiciousResults = $vendorResults | + Where-Object -Property category -EQ 'suspicious' | + Select-Object -ExpandProperty engine_name [PSCustomObject] @{ 'App.Name' = $app 'App.Hash' = $hash 'App.Size (MB)' = $fileSize.ToString('0.00') FileReport = $report_url - Malicious = $malicious - Suspicious = $suspicious + Malicious = if ($maliciousResults) { $maliciousResults } else { 0 } + Suspicious = if ($suspiciousResults) { $suspiciousResults } else { 0 } Timeout = $timeout Undetected = $undetected UrlReport = $null From e4207f489b33b8b9791387a7e7c4b1941c1eaee5 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Mon, 13 Jun 2022 22:38:42 +0800 Subject: [PATCH 09/28] adjust output --- libexec/scoop-virustotal.ps1 | 119 +++++++++++++++++------------------ 1 file changed, 57 insertions(+), 62 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index be83fe7d81..33c40554e2 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -40,7 +40,7 @@ $architecture = ensure_architecture ($opt.a + $opt.arch) if (is_scoop_outdated) { if ($opt.u -or $opt.'no-update-scoop') { - Write-Warning 'Scoop is out of date.' + warn 'Scoop is out of date.' } else { scoop update } @@ -67,9 +67,8 @@ $exit_code = 0 # Global API key: $api_key = get_config virustotal_api_key if (!$api_key) { - Write-Warning ("VirusTotal API key is not configured`n`n" + - "`tscoop config virustotal_api_key ") - exit $_ERR_NO_API_KEY + abort ("VirusTotal API key is not configured`n`n" + + "`tscoop config virustotal_api_key ") $_ERR_NO_API_KEY } # Global flag to explain only once about sleep between requests @@ -89,9 +88,7 @@ Function ConvertTo-VirusTotalUrlId ($url) { Function Get-RemoteFileSize ($url) { $response = Invoke-WebRequest -Uri $url -Method HEAD -UseBasicParsing - $response.Headers.'Content-Length' | ForEach-Object { - ([System.Convert]::ToDouble($_) / 1048576) - } + $response.Headers.'Content-Length' | ForEach-Object { [System.Convert]::ToInt32($_) } } Function Get-VirusTotalResultByHash ($hash, $app) { @@ -109,31 +106,31 @@ Function Get-VirusTotalResultByHash ($hash, $app) { [int]$undetected = json_path $stats '$.undetected' [int]$unsafe = $malicious + $suspicious [int]$total = $unsafe + $undetected - $fileSize = ([System.Convert]::ToDouble((json_path $result '$.data.attributes.size')) / 1048576) + [int]$fileSize = json_path $result '$.data.attributes.size' $report_url = "https://www.virustotal.com/gui/file/$hash" if ($total -eq 0) { - Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' + info "$app`: Analysis in progress." [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $hash - 'App.Size (MB)' = $fileSize.ToString('0.00') - FileReport = $report_url - UrlReport = $null + 'App.Name' = $app + 'App.Hash' = $hash + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $report_url + 'UrlReport.Url' = $null } } else { $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value switch ($unsafe) { 0 { - Write-Information "$($PSStyle.Foreground.BrightGreen)INFO : $app`: $unsafe/$total$($PSStyle.Reset)" -InformationAction 'Continue' + success "$app`: $unsafe/$total" } 1 { - Write-Warning "$app`: $unsafe/$total" + warn "$app`: $unsafe/$total" } 2 { - Write-Warning "$app`: $unsafe/$total" + warn "$app`: $unsafe/$total" } Default { - Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning)/$total" + warn "$app`: $unsafe/$total" } } $maliciousResults = $vendorResults | @@ -143,15 +140,15 @@ Function Get-VirusTotalResultByHash ($hash, $app) { Where-Object -Property category -EQ 'suspicious' | Select-Object -ExpandProperty engine_name [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $hash - 'App.Size (MB)' = $fileSize.ToString('0.00') - FileReport = $report_url - Malicious = if ($maliciousResults) { $maliciousResults } else { 0 } - Suspicious = if ($suspiciousResults) { $suspiciousResults } else { 0 } - Timeout = $timeout - Undetected = $undetected - UrlReport = $null + 'App.Name' = $app + 'App.Hash' = $hash + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $report_url + 'FileReport.Malicious' = if ($maliciousResults) { $maliciousResults } else { 0 } + 'FileReport.Suspicious' = if ($suspiciousResults) { $suspiciousResults } else { 0 } + 'FileReport.Timeout' = $timeout + 'FileReport.Undetected' = $undetected + 'UrlReport.Url' = $null } } if ($unsafe -gt 0) { @@ -170,23 +167,23 @@ Function Get-VirusTotalResultByUrl ($url, $app) { $id = json_path $result '$.data.id' $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null $url_report_url = "https://www.virustotal.com/gui/url/$id" - Write-Information "INFO : $app`: Url report found." + info "$app`: Url report found." if (!$hash) { - Write-Information "INFO : $app`: Related file report not found." - Write-Warning "$app`: Manual file upload is required (instead of url submission)." + info "$app`: Related file report not found." + warn "$app`: Manual file upload is required (instead of url submission)." [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $null - FileReport = $null - UrlReport = $url_report_url + 'App.Name' = $app + 'App.Hash' = $null + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url } } else { - Write-Information "INFO : $app`: Related file report found." + info "$app`: Related file report found." [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $hash - FileReport = $null - UrlReport = $url_report_url + 'App.Name' = $app + 'App.Hash' = $hash + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url } } } @@ -203,7 +200,7 @@ Function Get-VirusTotalResultByUrl ($url, $app) { # overflow) if the submission keeps failing. Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { if (!$do_scan) { - Write-Warning "$app`: not found`: you can manually submit $url" + warn "$app`: not found`: you can manually submit $url" return } @@ -223,15 +220,15 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { $id = ((json_path $result '$.data.id') -split '-')[1] $url_report_url = "https://www.virustotal.com/gui/url/$id" $fileSize = Get-RemoteFileSize $url - if ($fileSize -gt 70) { - Write-Information "INFO : $app`: Remote file size: $($fileSize.ToString('0.00 MB')). Large files might require manual file upload instead of url submission." -InformationAction 'Continue' + if ($fileSize -gt 80000000) { + info "$app`: Remote file size: $(filesize $fileSize). Large files might require manual file upload instead of url submission." } - Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' + info "$app`: Analysis in progress." [PSCustomObject] @{ - 'App.Name' = $app - 'App.Size (MB)' = $fileSize.ToString('0.00') - FileReport = $null - UrlReport = $url_report_url + 'App.Name' = $app + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url } return } @@ -240,16 +237,16 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { if (!$retrying) { if (!$explained_rate_limit_sleeping) { $explained_rate_limit_sleeping = $True - Write-Information "INFO : Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" -InformationAction 'Continue' + info "Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" } Start-Sleep -s (60 + $requests) Submit-ToVirusTotal $url $app $do_scan $True } else { - Write-Warning "$app`: VirusTotal submission of $url failed`:`n" + + warn "$app`: VirusTotal submission of $url failed`:`n" + "`tAPI returned $($result.StatusCode) after retrying" } } catch [Exception] { - Write-Warning "$app`: VirusTotal submission failed`: $($_.Exception.Message)" + warn "$app`: VirusTotal submission failed`: $($_.Exception.Message)" return } } @@ -259,7 +256,7 @@ $reports = $apps | ForEach-Object { $null, $manifest, $bucket, $null = Get-Manifest $app if (!$manifest) { $exit_code = $exit_code -bor $_ERR_NO_INFO - Write-Warning "$app`: manifest not found" + warn "$app`: manifest not found" return } @@ -275,7 +272,7 @@ $reports = $apps | ForEach-Object { if ($matches.algo -notmatch '(md5|sha1|sha256)') { $hash = $null $isHashUnsupported = $true - Write-Warning "$app`: Unsupported hash $($matches.algo). Will search by url instead." + warn "$app`: Unsupported hash $($matches.algo). Will search by url instead." } } if ($hash) { @@ -283,18 +280,17 @@ $reports = $apps | ForEach-Object { $file_report return } elseif (!$isHashUnsupported) { - Write-Warning "$app`: Hash not found. Will search by url instead." + warn "$app`: Hash not found. Will search by url instead." } } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION if ($_.Exception.Response.StatusCode -eq 404) { - Write-Warning "$app`: File report not found. Will search by url instead." + warn "$app`: File report not found. Will search by url instead." } else { if ($_.Exception.Response.StatusCode -in 204, 429) { - Write-Error "$app`: VirusTotal request failed`: $($_.Exception.Message)" - exit $exit_code + abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code } - Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" + warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" return } } @@ -304,7 +300,7 @@ $reports = $apps | ForEach-Object { $hash = $url_report.'App.Hash' if ($hash) { $file_report = Get-VirusTotalResultByHash $hash $app - $file_report.'UrlReport' = $url_report.'UrlReport' + $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' $file_report } else { $url_report @@ -312,14 +308,13 @@ $reports = $apps | ForEach-Object { } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION if ($_.Exception.Response.StatusCode -eq 404) { - Write-Warning "$app`: Url report not found. Will submit $url" + warn "$app`: Url report not found. Will submit $url" Submit-ToVirusTotal $url $app ($opt.scan -or $opt.s) } else { if ($_.Exception.Response.StatusCode -in 204, 429) { - Write-Error "$app`: VirusTotal request failed`: $($_.Exception.Message)" - exit $exit_code + abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code } - Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" + warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" return } } From e3249edfd77849db459ef0e0c64420210b45cb61 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Mon, 13 Jun 2022 23:15:55 +0800 Subject: [PATCH 10/28] fix for some rare cases --- libexec/scoop-virustotal.ps1 | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 33c40554e2..de3af7faef 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -166,11 +166,16 @@ Function Get-VirusTotalResultByUrl ($url, $app) { $result = $response.Content $id = json_path $result '$.data.id' $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null + $last_analysis_date = json_path $result '$.data.attributes.last_analysis_date' 6>$null $url_report_url = "https://www.virustotal.com/gui/url/$id" info "$app`: Url report found." if (!$hash) { - info "$app`: Related file report not found." - warn "$app`: Manual file upload is required (instead of url submission)." + if (!$last_analysis_date) { + info "$app`: Analysis in progress." + } else { + info "$app`: Related file report not found." + warn "$app`: Manual file upload is required (instead of url submission)." + } [PSCustomObject] @{ 'App.Name' = $app 'App.Hash' = $null @@ -285,6 +290,7 @@ $reports = $apps | ForEach-Object { } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION if ($_.Exception.Response.StatusCode -eq 404) { + $file_report_not_found = $true warn "$app`: File report not found. Will search by url instead." } else { if ($_.Exception.Response.StatusCode -in 204, 429) { @@ -298,7 +304,9 @@ $reports = $apps | ForEach-Object { try { $url_report = Get-VirusTotalResultByUrl $url $app $hash = $url_report.'App.Hash' - if ($hash) { + if ($hash -and ($file_report_not_found -eq $true)) { + error "$app`: Hash not matched" + } elseif ($hash) { $file_report = Get-VirusTotalResultByHash $hash $app $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' $file_report From 301c7e958d8f684a9c6e04406d5f476504fe4979 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Mon, 13 Jun 2022 23:38:00 +0800 Subject: [PATCH 11/28] add '--passthru' parameter --- libexec/scoop-virustotal.ps1 | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index de3af7faef..51ca752350 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -26,6 +26,7 @@ # your virustotal_api_key. # -n, --no-depends By default, all dependencies are checked too. This flag avoids it. # -u, --no-update-scoop Don't update Scoop before checking if it's outdated +# -p, --passthru Returns report . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' @@ -33,7 +34,7 @@ . "$PSScriptRoot\..\lib\install.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\depends.ps1" # 'Get-Dependency' -$opt, $apps, $err = getopt $args 'a:snu' @('arch=', 'scan', 'no-depends', 'no-update-scoop') +$opt, $apps, $err = getopt $args 'a:snu' @('arch=', 'scan', 'no-depends', 'no-update-scoop', 'passthru') if ($err) { "scoop virustotal: $err"; exit 1 } if (!$apps) { my_usage; exit 1 } $architecture = ensure_architecture ($opt.a + $opt.arch) @@ -121,16 +122,16 @@ Function Get-VirusTotalResultByHash ($hash, $app) { $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value switch ($unsafe) { 0 { - success "$app`: $unsafe/$total" + success "$app`: $unsafe/$total, see $report_url" } 1 { - warn "$app`: $unsafe/$total" + warn "$app`: $unsafe/$total, see $report_url" } 2 { - warn "$app`: $unsafe/$total" + warn "$app`: $unsafe/$total, see $report_url" } Default { - warn "$app`: $unsafe/$total" + warn "`e[31m$app`: $unsafe/$total, see $report_url`e[0m" } } $maliciousResults = $vendorResults | @@ -328,6 +329,8 @@ $reports = $apps | ForEach-Object { } } } -$reports +if ($opt.p -or $opt.'passthru') { + $reports +} exit $exit_code From a92525210eafb4149289f0090dea8b4fbbbabbe6 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 00:43:38 +0800 Subject: [PATCH 12/28] feat(scoop-virustotal): migrate to virustotal api v3, improve flows, and use standard psstreams --- libexec/scoop-virustotal.ps1 | 286 +++++++++++++++++++++++------------ 1 file changed, 187 insertions(+), 99 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index cd2aa328f1..fdafae470b 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -1,28 +1,21 @@ # Usage: scoop virustotal [* | app1 app2 ...] [options] -# Summary: Look for app's hash on virustotal.com -# Help: Look for app's hash (MD5, SHA1 or SHA256) on virustotal.com +# Summary: Look for app's hash or url on virustotal.com +# Help: Look for app's hash or url on virustotal.com # # Use a single '*' for app to check all installed apps. # -# The download's hash is also a key to access VirusTotal's scan results. -# This allows to check the safety of the files without even downloading -# them in many cases. If the hash is unknown to VirusTotal, the -# download link is printed to submit it to VirusTotal. -# -# If you have signed up to VirusTotal's community, you have an API key -# that this script can use to submit unknown packages for inspection -# if you use the `--scan' flag. Tell scoop about your API key with: +# To use this command, you have to sign up to VirusTotal's community, +# and get an API key. Then, tell scoop about your API key with: # # scoop config virustotal_api_key # # Exit codes: -# 0 -> success -# 1 -> problem parsing arguments -# 2 -> at least one package was marked unsafe by VirusTotal -# 4 -> at least one exception was raised while looking for info -# 8 -> at least one package couldn't be queried because its hash type -# isn't supported by VirusTotal, the manifest couldn't be found -# or didn't contain a hash +# 0 -> success +# 1 -> problem parsing arguments +# 2 -> at least one package was marked unsafe by VirusTotal +# 4 -> at least one exception was raised while looking for info +# 8 -> at least one package couldn't be queried because the manifest couldn't be found +# 16 -> VirusTotal API key is not configured # Note: the exit codes (2, 4 & 8) may be combined, e.g. 6 -> exit codes # 2 & 4 combined # @@ -67,11 +60,17 @@ if (!$opt.n -and !$opt.'no-depends') { $_ERR_UNSAFE = 2 $_ERR_EXCEPTION = 4 $_ERR_NO_INFO = 8 +$_ERR_NO_API_KEY = 16 $exit_code = 0 -# Global flag to warn only once about missing API key: -$warned_no_api_key = $False +# Global API key: +$api_key = get_config virustotal_api_key +if (!$api_key) { + Write-Warning ("VirusTotal API key is not configured`n`n" + + "`tscoop config virustotal_api_key ") + exit $_ERR_NO_API_KEY +} # Global flag to explain only once about sleep between requests $explained_rate_limit_sleeping = $False @@ -80,66 +79,124 @@ $explained_rate_limit_sleeping = $False # script execution progresses $requests = 0 -Function Get-VirusTotalResult($hash, $app) { +Function ConvertTo-VirusTotalUrlId ($url) { + $url_id = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($url)) + $url_id = $url_id -replace '\+', '-' + $url_id = $url_id -replace '/', '_' + $url_id = $url_id -replace '=', '' + $url_id +} + +Function Get-RemoteFileSize ($url) { + $response = Invoke-WebRequest -Uri $url -Method HEAD -UseBasicParsing + $response.Headers.'Content-Length' | ForEach-Object { + ([System.Convert]::ToDouble($_) / 1048576) + } +} + +Function Get-VirusTotalResultByHash ($hash, $app) { $hash = $hash.ToLower() - $url = "https://www.virustotal.com/ui/files/$hash" - $wc = New-Object Net.Webclient - $wc.Headers.Add('User-Agent', (Get-UserAgent)) - $data = $wc.DownloadData($url) - $result = (Get-Encoding($wc)).GetString($data) + $api_url = "https://www.virustotal.com/api/v3/files/$hash" + $headers = @{} + $headers.Add('Accept', 'application/json') + $headers.Add('x-apikey', $api_key) + $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing + $result = $response.Content $stats = json_path $result '$.data.attributes.last_analysis_stats' - $malicious = json_path $stats '$.malicious' - $suspicious = json_path $stats '$.suspicious' - $undetected = json_path $stats '$.undetected' - $unsafe = [int]$malicious + [int]$suspicious - $see_url = "see https://www.virustotal.com/#/file/$hash/detection" - switch ($unsafe) { - 0 { if ($undetected -eq 0) { $fg = 'Yellow' } else { $fg = 'DarkGreen' } } - 1 { $fg = 'DarkYellow' } - 2 { $fg = 'Yellow' } - default { $fg = 'Red' } + [int]$malicious = json_path $stats '$.malicious' + [int]$suspicious = json_path $stats '$.suspicious' + [int]$timeout = json_path $stats '$.timeout' + [int]$undetected = json_path $stats '$.undetected' + [int]$unsafe = $malicious + $suspicious + [int]$total = $unsafe + $undetected + $report_url = "https://www.virustotal.com/gui/file/$hash" + if ($undetected -eq 0) { + Write-Information "INFO : $app`: Analysis is in progress." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Hash' = $hash + FileReport = $report_url + } + } else { + switch ($unsafe) { + 0 { + Write-Information "$($PSStyle.Foreground.BrightGreen)INFO : $app`: $unsafe / $total$($PSStyle.Reset)" -InformationAction 'Continue' + } + 1 { + Write-Warning "$app`: $unsafe / $total" + } + 2 { + Write-Warning "$app`: $unsafe / $total" + } + Default { + Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning) / $total" + } + } + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Hash' = $hash + Malicious = $malicious + Suspicious = $suspicious + Timeout = $timeout + Undetected = $undetected + FileReport = $report_url + } } - Write-Host -f $fg "$app`: $unsafe/$undetected, $see_url" if ($unsafe -gt 0) { - return $_ERR_UNSAFE + $Script:exit_code = $exit_code -bor $_ERR_UNSAFE } - return 0 } -Function Search-VirusTotal ($hash, $app) { - if ($hash -match '(?[^:]+):(?.*)') { - $hash = $matches['hash'] - if ($matches['algo'] -match '(md5|sha1|sha256)') { - return Get-VirusTotalResult $hash $app - } else { - warn "$app`: Unsupported hash $($matches['algo']). VirusTotal needs md5, sha1 or sha256." - return $_ERR_NO_INFO +Function Get-VirusTotalResultByUrl ($url, $app) { + $id = ConvertTo-VirusTotalUrlId $url + $api_url = "https://www.virustotal.com/api/v3/urls/$id" + $headers = @{} + $headers.Add('Accept', 'application/json') + $headers.Add('x-apikey', $api_key) + $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing + $result = $response.Content + $id = json_path $result '$.data.id' + $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null + $url_report_url = "https://www.virustotal.com/gui/url/$id" + Write-Information "INFO : $app`: Url report found: $url_report_url" -InformationAction 'Continue' + Write-Information "INFO : $($PSStyle.Formatting.Warning)Url report is not accurate.$($PSStyle.Reset)" -InformationAction 'Continue' + if (!$hash) { + Write-Warning "$app`: File report not found. Manual file upload is required (instead of url submission)." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + UrlReport = $url_report_url + } + } else { + Write-Information "INFO : $app`: Related file report found." -InformationAction 'Continue' + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Hash' = $hash + 'App.Url' = $url + UrlReport = $url_report_url } } - - return Get-VirusTotalResult $hash $app } -Function Submit-RedirectedUrl { - # Follow up to one level of HTTP redirection - # - # Copied from http://www.powershellmagazine.com/2013/01/29/pstip-retrieve-a-redirected-url/ - # Adapted according to Roy's response (January 23, 2014 at 11:59 am) - # Adapted to always return an URL - Param ( - [Parameter(Mandatory = $true)] - [String]$URL - ) - $request = [System.Net.WebRequest]::Create($url) - $request.AllowAutoRedirect = $false - $response = $request.GetResponse() - if (([int]$response.StatusCode -ge 300) -and ([int]$response.StatusCode -lt 400)) { - $redir = $response.GetResponseHeader('Location') +Function Get-VirusTotalResult ($hash, $url, $app) { + if ($hash -match '(?[^:]+):(?.*)') { + $hash = $matches.hash + if ($matches.algo -match '(md5|sha1|sha256)') { + Get-VirusTotalResultByHash $hash $app + return + } + Write-Warning "$app`: Unsupported hash $($matches.algo), will search by url instead" + } elseif ($hash) { + Get-VirusTotalResultByHash $hash $app + return } else { - $redir = $URL + Write-Warning "$app`: Can't find hash for $url, will search by url instead" + } + $result = Get-VirusTotalResultByUrl $url $app + $hash = $result.'App.Hash' + if ($hash) { + Get-VirusTotalResultByHash $result.'App.Hash' $app } - $response.Close() - return $redir } # Submit-ToVirusTotal @@ -153,32 +210,32 @@ Function Submit-RedirectedUrl { # exceeded, without risking an infinite loop (as stack # overflow) if the submission keeps failing. Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { - $api_key = get_config virustotal_api_key - if ($do_scan -and !$api_key -and !$warned_no_api_key) { - $warned_no_api_key = $true - info 'Submitting unknown apps needs a VirusTotal API key. ' + - "Set it up with`n`tscoop config virustotal_api_key " - - } - if (!$do_scan -or !$api_key) { - warn "$app`: not found`: manually submit $url" + if (!$do_scan) { + warn "$app`: not found`: you can manually submit $url" return } try { - # Follow redirections (for e.g. sourceforge URLs) because - # VirusTotal analyzes only "direct" download links - $url = $url.Split('#').GetValue(0) - $new_redir = $url - do { - $orig_redir = $new_redir - $new_redir = Submit-RedirectedUrl $orig_redir - } while ($orig_redir -ne $new_redir) $requests += 1 - $result = Invoke-WebRequest -Uri 'https://www.virustotal.com/vtapi/v2/url/scan' -Body @{apikey = $api_key; url = $new_redir } -Method Post -UseBasicParsing - $submitted = $result.StatusCode -eq 200 - if ($submitted) { - warn "$app`: not found`: submitted $url" + + $encoded_url = [System.Web.HttpUtility]::UrlEncode($url) + $api_url = 'https://www.virustotal.com/api/v3/urls' + $content_type = 'application/x-www-form-urlencoded' + $headers = @{} + $headers.Add('Accept', 'application/json') + $headers.Add('x-apikey', $api_key) + $headers.Add('Content-Type', $content_type) + $body = "url=$encoded_url" + $result = Invoke-WebRequest -Uri $api_url -Method POST -Headers $headers -ContentType $content_type -Body $body -UseBasicParsing + if ($result.StatusCode -eq 200) { + $id = ((json_path $result '$.data.id') -split '-')[1] + $url_report_url = "https://www.virustotal.com/gui/url/$id" + $fileSize = Get-RemoteFileSize $url + Write-Information "INFO : $app`: Remote file size: $($fileSize.ToString('0.00 MB'))" -InformationAction 'Continue' + if ($fileSize -gt 70) { + Write-Information "INFO : $($PSStyle.Formatting.Warning)Large files might require manual file upload instead of url submission.$($PSStyle.Reset)" + } + Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' return } @@ -186,26 +243,26 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { if (!$retrying) { if (!$explained_rate_limit_sleeping) { $explained_rate_limit_sleeping = $True - info "Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" + Write-Information "INFO : Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" -InformationAction 'Continue' } Start-Sleep -s (60 + $requests) - Submit-ToVirusTotal $new_redir $app $do_scan $True + Submit-ToVirusTotal $url $app $do_scan $True } else { - warn "$app`: VirusTotal submission of $url failed`:`n" + + Write-Warning "$app`: VirusTotal submission of $url failed`:`n" + "`tAPI returned $($result.StatusCode) after retrying" } } catch [Exception] { - warn "$app`: VirusTotal submission failed`: $($_.Exception.Message)" + Write-Warning "$app`: VirusTotal submission failed`: $($_.Exception.Message)" return } } -foreach ($app in $apps) { - # write-host $app +$apps | ForEach-Object { + $app = $_ $null, $manifest, $bucket, $null = Get-Manifest $app if (!$manifest) { $exit_code = $exit_code -bor $_ERR_NO_INFO - warn "$app`: manifest not found" + Write-Warning "$app`: manifest not found" continue } @@ -215,20 +272,51 @@ foreach ($app in $apps) { $hash = hash_for_url $manifest $url $architecture try { + if ($hash -match '(?[^:]+):(?.*)') { + $hash = $matches.hash + if ($matches.algo -notmatch '(md5|sha1|sha256)') { + $hash = $null + Write-Warning "$app`: Unsupported hash $($matches.algo). Will search by url instead." + } + } if ($hash) { - $exit_code = $exit_code -bor (Search-VirusTotal $hash $app) + $report = Get-VirusTotalResultByHash $hash $app + $report + return } else { - warn "$app`: Can't find hash for $url" + Write-Warning "$app`: Hash for $url not found. Will search by url instead." + } + } catch [Exception] { + $exit_code = $exit_code -bor $_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + Write-Warning "$app`: File report not found. Will search by url instead." + } else { + if ($_.Exception.Response.StatusCode -in 204, 429) { + abort "$app`: VirusTotal request failed`: $($_.Exception.Message)", $exit_code + } + Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" + return + } + } + + try { + $url_report = Get-VirusTotalResultByUrl $url $app + $hash = $url_report.'App.Hash' + if ($hash) { + $report = Get-VirusTotalResultByHash $hash $app + $report } } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION - if ($_.Exception.Message -like '*(404)*') { + if ($_.Exception.Response.StatusCode -eq 404) { + Write-Warning "$app`: Url report not found. Will submit $url" Submit-ToVirusTotal $url $app ($opt.scan -or $opt.s) } else { - if ($_.Exception.Message -match '\(204|429\)') { + if ($_.Exception.Response.StatusCode -in 204, 429) { abort "$app`: VirusTotal request failed`: $($_.Exception.Message)", $exit_code } - warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" + Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" + return } } } From aa830bd45a008172f8bd7049ecea3aebb9dbe0b4 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 18:36:32 +0800 Subject: [PATCH 13/28] remove unused codes --- libexec/scoop-virustotal.ps1 | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index fdafae470b..55550f8215 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -178,27 +178,6 @@ Function Get-VirusTotalResultByUrl ($url, $app) { } } -Function Get-VirusTotalResult ($hash, $url, $app) { - if ($hash -match '(?[^:]+):(?.*)') { - $hash = $matches.hash - if ($matches.algo -match '(md5|sha1|sha256)') { - Get-VirusTotalResultByHash $hash $app - return - } - Write-Warning "$app`: Unsupported hash $($matches.algo), will search by url instead" - } elseif ($hash) { - Get-VirusTotalResultByHash $hash $app - return - } else { - Write-Warning "$app`: Can't find hash for $url, will search by url instead" - } - $result = Get-VirusTotalResultByUrl $url $app - $hash = $result.'App.Hash' - if ($hash) { - Get-VirusTotalResultByHash $result.'App.Hash' $app - } -} - # Submit-ToVirusTotal # - $url: where file to check can be downloaded # - $app: Name of the application (used for reporting) From 331f134357254c900926e8db1df6480d2326a49a Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 18:39:38 +0800 Subject: [PATCH 14/28] fix bug --- libexec/scoop-virustotal.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 55550f8215..df010aff87 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -242,7 +242,7 @@ $apps | ForEach-Object { if (!$manifest) { $exit_code = $exit_code -bor $_ERR_NO_INFO Write-Warning "$app`: manifest not found" - continue + return } $urls = script:url $manifest $architecture From 20d5f4772f8515ee160af5320841beebfc722e79 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 18:47:50 +0800 Subject: [PATCH 15/28] adjust output --- libexec/scoop-virustotal.ps1 | 59 ++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index df010aff87..fc8942f887 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -116,30 +116,36 @@ Function Get-VirusTotalResultByHash ($hash, $app) { 'App.Name' = $app 'App.Hash' = $hash FileReport = $report_url + Malicious = $null + Suspicious = $null + Timeout = $null + Undetected = $null + UrlReport = $null } } else { switch ($unsafe) { 0 { - Write-Information "$($PSStyle.Foreground.BrightGreen)INFO : $app`: $unsafe / $total$($PSStyle.Reset)" -InformationAction 'Continue' + Write-Information "$($PSStyle.Foreground.BrightGreen)INFO : $app`: $unsafe/$total$($PSStyle.Reset)" -InformationAction 'Continue' } 1 { - Write-Warning "$app`: $unsafe / $total" + Write-Warning "$app`: $unsafe/$total" } 2 { - Write-Warning "$app`: $unsafe / $total" + Write-Warning "$app`: $unsafe $total" } Default { - Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning) / $total" + Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning)/$total" } } [PSCustomObject] @{ 'App.Name' = $app 'App.Hash' = $hash + FileReport = $report_url Malicious = $malicious Suspicious = $suspicious Timeout = $timeout Undetected = $undetected - FileReport = $report_url + UrlReport = $null } } if ($unsafe -gt 0) { @@ -158,21 +164,29 @@ Function Get-VirusTotalResultByUrl ($url, $app) { $id = json_path $result '$.data.id' $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null $url_report_url = "https://www.virustotal.com/gui/url/$id" - Write-Information "INFO : $app`: Url report found: $url_report_url" -InformationAction 'Continue' - Write-Information "INFO : $($PSStyle.Formatting.Warning)Url report is not accurate.$($PSStyle.Reset)" -InformationAction 'Continue' + Write-Information "INFO : $app`: Url report found." if (!$hash) { - Write-Warning "$app`: File report not found. Manual file upload is required (instead of url submission)." + Write-Information "INFO : $app`: Related file report not found." + Write-Warning "$app`: Manual file upload is required (instead of url submission)." [PSCustomObject] @{ 'App.Name' = $app - 'App.Url' = $url + FileReport = $null + Malicious = $null + Suspicious = $null + Timeout = $null + Undetected = $null UrlReport = $url_report_url } } else { - Write-Information "INFO : $app`: Related file report found." -InformationAction 'Continue' + Write-Information "INFO : $app`: Related file report found." [PSCustomObject] @{ 'App.Name' = $app 'App.Hash' = $hash - 'App.Url' = $url + FileReport = $null + Malicious = $null + Suspicious = $null + Timeout = $null + Undetected = $null UrlReport = $url_report_url } } @@ -210,9 +224,8 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { $id = ((json_path $result '$.data.id') -split '-')[1] $url_report_url = "https://www.virustotal.com/gui/url/$id" $fileSize = Get-RemoteFileSize $url - Write-Information "INFO : $app`: Remote file size: $($fileSize.ToString('0.00 MB'))" -InformationAction 'Continue' if ($fileSize -gt 70) { - Write-Information "INFO : $($PSStyle.Formatting.Warning)Large files might require manual file upload instead of url submission.$($PSStyle.Reset)" + Write-Information "INFO : $app`: Remote file size: $($fileSize.ToString('0.00 MB')). Large files might require manual file upload instead of url submission." -InformationAction 'Continue' } Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' return @@ -236,7 +249,7 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { } } -$apps | ForEach-Object { +$reports = $apps | ForEach-Object { $app = $_ $null, $manifest, $bucket, $null = Get-Manifest $app if (!$manifest) { @@ -251,19 +264,21 @@ $apps | ForEach-Object { $hash = hash_for_url $manifest $url $architecture try { + $isHashUnsupported = $false if ($hash -match '(?[^:]+):(?.*)') { $hash = $matches.hash if ($matches.algo -notmatch '(md5|sha1|sha256)') { $hash = $null + $isHashUnsupported = $true Write-Warning "$app`: Unsupported hash $($matches.algo). Will search by url instead." } } if ($hash) { - $report = Get-VirusTotalResultByHash $hash $app - $report + $file_report = Get-VirusTotalResultByHash $hash $app + $file_report return - } else { - Write-Warning "$app`: Hash for $url not found. Will search by url instead." + } elseif (!$isHashUnsupported) { + Write-Warning "$app`: Hash not found. Will search by url instead." } } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION @@ -282,8 +297,11 @@ $apps | ForEach-Object { $url_report = Get-VirusTotalResultByUrl $url $app $hash = $url_report.'App.Hash' if ($hash) { - $report = Get-VirusTotalResultByHash $hash $app - $report + $file_report = Get-VirusTotalResultByHash $hash $app + $file_report.'UrlReport' = $url_report.'UrlReport' + $file_report + } else { + $url_report } } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION @@ -300,5 +318,6 @@ $apps | ForEach-Object { } } } +$reports exit $exit_code From 5a20a6c8a5ed2108b74d2b79ba181fa537aad69a Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 19:09:37 +0800 Subject: [PATCH 16/28] fix --- libexec/scoop-virustotal.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index fc8942f887..b74efecdf8 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -110,7 +110,7 @@ Function Get-VirusTotalResultByHash ($hash, $app) { [int]$unsafe = $malicious + $suspicious [int]$total = $unsafe + $undetected $report_url = "https://www.virustotal.com/gui/file/$hash" - if ($undetected -eq 0) { + if ($total -eq 0) { Write-Information "INFO : $app`: Analysis is in progress." [PSCustomObject] @{ 'App.Name' = $app From 1fc1cbb362a8c0c12d7933efba57f1f43c004152 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sat, 11 Jun 2022 19:25:41 +0800 Subject: [PATCH 17/28] fix output --- libexec/scoop-virustotal.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index b74efecdf8..6f77d60b74 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -131,7 +131,7 @@ Function Get-VirusTotalResultByHash ($hash, $app) { Write-Warning "$app`: $unsafe/$total" } 2 { - Write-Warning "$app`: $unsafe $total" + Write-Warning "$app`: $unsafe/$total" } Default { Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning)/$total" From 9d9fc9cd1cd73fc348275728d953245bf03b1dad Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sun, 12 Jun 2022 15:18:37 +0800 Subject: [PATCH 18/28] Adjust output --- libexec/scoop-virustotal.ps1 | 58 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 6f77d60b74..f747b91443 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -40,7 +40,7 @@ $architecture = ensure_architecture ($opt.a + $opt.arch) if (is_scoop_outdated) { if ($opt.u -or $opt.'no-update-scoop') { - warn 'Scoop is out of date.' + Write-Warning 'Scoop is out of date.' } else { scoop update } @@ -109,18 +109,16 @@ Function Get-VirusTotalResultByHash ($hash, $app) { [int]$undetected = json_path $stats '$.undetected' [int]$unsafe = $malicious + $suspicious [int]$total = $unsafe + $undetected + $fileSize = ([System.Convert]::ToDouble((json_path $result '$.data.attributes.size')) / 1048576) $report_url = "https://www.virustotal.com/gui/file/$hash" if ($total -eq 0) { - Write-Information "INFO : $app`: Analysis is in progress." + Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $hash - FileReport = $report_url - Malicious = $null - Suspicious = $null - Timeout = $null - Undetected = $null - UrlReport = $null + 'App.Name' = $app + 'App.Hash' = $hash + 'App.Size (MB)' = $fileSize.ToString('0.00') + FileReport = $report_url + UrlReport = $null } } else { switch ($unsafe) { @@ -138,14 +136,15 @@ Function Get-VirusTotalResultByHash ($hash, $app) { } } [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $hash - FileReport = $report_url - Malicious = $malicious - Suspicious = $suspicious - Timeout = $timeout - Undetected = $undetected - UrlReport = $null + 'App.Name' = $app + 'App.Hash' = $hash + 'App.Size (MB)' = $fileSize.ToString('0.00') + FileReport = $report_url + Malicious = $malicious + Suspicious = $suspicious + Timeout = $timeout + Undetected = $undetected + UrlReport = $null } } if ($unsafe -gt 0) { @@ -170,11 +169,8 @@ Function Get-VirusTotalResultByUrl ($url, $app) { Write-Warning "$app`: Manual file upload is required (instead of url submission)." [PSCustomObject] @{ 'App.Name' = $app + 'App.Hash' = $null FileReport = $null - Malicious = $null - Suspicious = $null - Timeout = $null - Undetected = $null UrlReport = $url_report_url } } else { @@ -183,10 +179,6 @@ Function Get-VirusTotalResultByUrl ($url, $app) { 'App.Name' = $app 'App.Hash' = $hash FileReport = $null - Malicious = $null - Suspicious = $null - Timeout = $null - Undetected = $null UrlReport = $url_report_url } } @@ -204,7 +196,7 @@ Function Get-VirusTotalResultByUrl ($url, $app) { # overflow) if the submission keeps failing. Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { if (!$do_scan) { - warn "$app`: not found`: you can manually submit $url" + Write-Warning "$app`: not found`: you can manually submit $url" return } @@ -228,6 +220,12 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { Write-Information "INFO : $app`: Remote file size: $($fileSize.ToString('0.00 MB')). Large files might require manual file upload instead of url submission." -InformationAction 'Continue' } Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Size (MB)' = $fileSize.ToString('0.00') + FileReport = $null + UrlReport = $url_report_url + } return } @@ -286,7 +284,8 @@ $reports = $apps | ForEach-Object { Write-Warning "$app`: File report not found. Will search by url instead." } else { if ($_.Exception.Response.StatusCode -in 204, 429) { - abort "$app`: VirusTotal request failed`: $($_.Exception.Message)", $exit_code + Write-Error "$app`: VirusTotal request failed`: $($_.Exception.Message)" + exit $exit_code } Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" return @@ -310,7 +309,8 @@ $reports = $apps | ForEach-Object { Submit-ToVirusTotal $url $app ($opt.scan -or $opt.s) } else { if ($_.Exception.Response.StatusCode -in 204, 429) { - abort "$app`: VirusTotal request failed`: $($_.Exception.Message)", $exit_code + Write-Error "$app`: VirusTotal request failed`: $($_.Exception.Message)" + exit $exit_code } Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" return From 36d5ce39161c91470cf6d4b4a5b1b24df50f04c8 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Sun, 12 Jun 2022 21:22:34 +0800 Subject: [PATCH 19/28] show vendors directly if malicious/suspicious != 0 --- libexec/scoop-virustotal.ps1 | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index f747b91443..be83fe7d81 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -121,6 +121,7 @@ Function Get-VirusTotalResultByHash ($hash, $app) { UrlReport = $null } } else { + $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value switch ($unsafe) { 0 { Write-Information "$($PSStyle.Foreground.BrightGreen)INFO : $app`: $unsafe/$total$($PSStyle.Reset)" -InformationAction 'Continue' @@ -135,13 +136,19 @@ Function Get-VirusTotalResultByHash ($hash, $app) { Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning)/$total" } } + $maliciousResults = $vendorResults | + Where-Object -Property category -EQ 'malicious' | + Select-Object -ExpandProperty engine_name + $suspiciousResults = $vendorResults | + Where-Object -Property category -EQ 'suspicious' | + Select-Object -ExpandProperty engine_name [PSCustomObject] @{ 'App.Name' = $app 'App.Hash' = $hash 'App.Size (MB)' = $fileSize.ToString('0.00') FileReport = $report_url - Malicious = $malicious - Suspicious = $suspicious + Malicious = if ($maliciousResults) { $maliciousResults } else { 0 } + Suspicious = if ($suspiciousResults) { $suspiciousResults } else { 0 } Timeout = $timeout Undetected = $undetected UrlReport = $null From 152f21acfef3247fe10d1f7e07400bb5a52be79d Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Mon, 13 Jun 2022 22:38:42 +0800 Subject: [PATCH 20/28] adjust output --- libexec/scoop-virustotal.ps1 | 119 +++++++++++++++++------------------ 1 file changed, 57 insertions(+), 62 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index be83fe7d81..33c40554e2 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -40,7 +40,7 @@ $architecture = ensure_architecture ($opt.a + $opt.arch) if (is_scoop_outdated) { if ($opt.u -or $opt.'no-update-scoop') { - Write-Warning 'Scoop is out of date.' + warn 'Scoop is out of date.' } else { scoop update } @@ -67,9 +67,8 @@ $exit_code = 0 # Global API key: $api_key = get_config virustotal_api_key if (!$api_key) { - Write-Warning ("VirusTotal API key is not configured`n`n" + - "`tscoop config virustotal_api_key ") - exit $_ERR_NO_API_KEY + abort ("VirusTotal API key is not configured`n`n" + + "`tscoop config virustotal_api_key ") $_ERR_NO_API_KEY } # Global flag to explain only once about sleep between requests @@ -89,9 +88,7 @@ Function ConvertTo-VirusTotalUrlId ($url) { Function Get-RemoteFileSize ($url) { $response = Invoke-WebRequest -Uri $url -Method HEAD -UseBasicParsing - $response.Headers.'Content-Length' | ForEach-Object { - ([System.Convert]::ToDouble($_) / 1048576) - } + $response.Headers.'Content-Length' | ForEach-Object { [System.Convert]::ToInt32($_) } } Function Get-VirusTotalResultByHash ($hash, $app) { @@ -109,31 +106,31 @@ Function Get-VirusTotalResultByHash ($hash, $app) { [int]$undetected = json_path $stats '$.undetected' [int]$unsafe = $malicious + $suspicious [int]$total = $unsafe + $undetected - $fileSize = ([System.Convert]::ToDouble((json_path $result '$.data.attributes.size')) / 1048576) + [int]$fileSize = json_path $result '$.data.attributes.size' $report_url = "https://www.virustotal.com/gui/file/$hash" if ($total -eq 0) { - Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' + info "$app`: Analysis in progress." [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $hash - 'App.Size (MB)' = $fileSize.ToString('0.00') - FileReport = $report_url - UrlReport = $null + 'App.Name' = $app + 'App.Hash' = $hash + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $report_url + 'UrlReport.Url' = $null } } else { $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value switch ($unsafe) { 0 { - Write-Information "$($PSStyle.Foreground.BrightGreen)INFO : $app`: $unsafe/$total$($PSStyle.Reset)" -InformationAction 'Continue' + success "$app`: $unsafe/$total" } 1 { - Write-Warning "$app`: $unsafe/$total" + warn "$app`: $unsafe/$total" } 2 { - Write-Warning "$app`: $unsafe/$total" + warn "$app`: $unsafe/$total" } Default { - Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning)/$total" + warn "$app`: $unsafe/$total" } } $maliciousResults = $vendorResults | @@ -143,15 +140,15 @@ Function Get-VirusTotalResultByHash ($hash, $app) { Where-Object -Property category -EQ 'suspicious' | Select-Object -ExpandProperty engine_name [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $hash - 'App.Size (MB)' = $fileSize.ToString('0.00') - FileReport = $report_url - Malicious = if ($maliciousResults) { $maliciousResults } else { 0 } - Suspicious = if ($suspiciousResults) { $suspiciousResults } else { 0 } - Timeout = $timeout - Undetected = $undetected - UrlReport = $null + 'App.Name' = $app + 'App.Hash' = $hash + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $report_url + 'FileReport.Malicious' = if ($maliciousResults) { $maliciousResults } else { 0 } + 'FileReport.Suspicious' = if ($suspiciousResults) { $suspiciousResults } else { 0 } + 'FileReport.Timeout' = $timeout + 'FileReport.Undetected' = $undetected + 'UrlReport.Url' = $null } } if ($unsafe -gt 0) { @@ -170,23 +167,23 @@ Function Get-VirusTotalResultByUrl ($url, $app) { $id = json_path $result '$.data.id' $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null $url_report_url = "https://www.virustotal.com/gui/url/$id" - Write-Information "INFO : $app`: Url report found." + info "$app`: Url report found." if (!$hash) { - Write-Information "INFO : $app`: Related file report not found." - Write-Warning "$app`: Manual file upload is required (instead of url submission)." + info "$app`: Related file report not found." + warn "$app`: Manual file upload is required (instead of url submission)." [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $null - FileReport = $null - UrlReport = $url_report_url + 'App.Name' = $app + 'App.Hash' = $null + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url } } else { - Write-Information "INFO : $app`: Related file report found." + info "$app`: Related file report found." [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $hash - FileReport = $null - UrlReport = $url_report_url + 'App.Name' = $app + 'App.Hash' = $hash + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url } } } @@ -203,7 +200,7 @@ Function Get-VirusTotalResultByUrl ($url, $app) { # overflow) if the submission keeps failing. Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { if (!$do_scan) { - Write-Warning "$app`: not found`: you can manually submit $url" + warn "$app`: not found`: you can manually submit $url" return } @@ -223,15 +220,15 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { $id = ((json_path $result '$.data.id') -split '-')[1] $url_report_url = "https://www.virustotal.com/gui/url/$id" $fileSize = Get-RemoteFileSize $url - if ($fileSize -gt 70) { - Write-Information "INFO : $app`: Remote file size: $($fileSize.ToString('0.00 MB')). Large files might require manual file upload instead of url submission." -InformationAction 'Continue' + if ($fileSize -gt 80000000) { + info "$app`: Remote file size: $(filesize $fileSize). Large files might require manual file upload instead of url submission." } - Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue' + info "$app`: Analysis in progress." [PSCustomObject] @{ - 'App.Name' = $app - 'App.Size (MB)' = $fileSize.ToString('0.00') - FileReport = $null - UrlReport = $url_report_url + 'App.Name' = $app + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url } return } @@ -240,16 +237,16 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { if (!$retrying) { if (!$explained_rate_limit_sleeping) { $explained_rate_limit_sleeping = $True - Write-Information "INFO : Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" -InformationAction 'Continue' + info "Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" } Start-Sleep -s (60 + $requests) Submit-ToVirusTotal $url $app $do_scan $True } else { - Write-Warning "$app`: VirusTotal submission of $url failed`:`n" + + warn "$app`: VirusTotal submission of $url failed`:`n" + "`tAPI returned $($result.StatusCode) after retrying" } } catch [Exception] { - Write-Warning "$app`: VirusTotal submission failed`: $($_.Exception.Message)" + warn "$app`: VirusTotal submission failed`: $($_.Exception.Message)" return } } @@ -259,7 +256,7 @@ $reports = $apps | ForEach-Object { $null, $manifest, $bucket, $null = Get-Manifest $app if (!$manifest) { $exit_code = $exit_code -bor $_ERR_NO_INFO - Write-Warning "$app`: manifest not found" + warn "$app`: manifest not found" return } @@ -275,7 +272,7 @@ $reports = $apps | ForEach-Object { if ($matches.algo -notmatch '(md5|sha1|sha256)') { $hash = $null $isHashUnsupported = $true - Write-Warning "$app`: Unsupported hash $($matches.algo). Will search by url instead." + warn "$app`: Unsupported hash $($matches.algo). Will search by url instead." } } if ($hash) { @@ -283,18 +280,17 @@ $reports = $apps | ForEach-Object { $file_report return } elseif (!$isHashUnsupported) { - Write-Warning "$app`: Hash not found. Will search by url instead." + warn "$app`: Hash not found. Will search by url instead." } } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION if ($_.Exception.Response.StatusCode -eq 404) { - Write-Warning "$app`: File report not found. Will search by url instead." + warn "$app`: File report not found. Will search by url instead." } else { if ($_.Exception.Response.StatusCode -in 204, 429) { - Write-Error "$app`: VirusTotal request failed`: $($_.Exception.Message)" - exit $exit_code + abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code } - Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" + warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" return } } @@ -304,7 +300,7 @@ $reports = $apps | ForEach-Object { $hash = $url_report.'App.Hash' if ($hash) { $file_report = Get-VirusTotalResultByHash $hash $app - $file_report.'UrlReport' = $url_report.'UrlReport' + $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' $file_report } else { $url_report @@ -312,14 +308,13 @@ $reports = $apps | ForEach-Object { } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION if ($_.Exception.Response.StatusCode -eq 404) { - Write-Warning "$app`: Url report not found. Will submit $url" + warn "$app`: Url report not found. Will submit $url" Submit-ToVirusTotal $url $app ($opt.scan -or $opt.s) } else { if ($_.Exception.Response.StatusCode -in 204, 429) { - Write-Error "$app`: VirusTotal request failed`: $($_.Exception.Message)" - exit $exit_code + abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code } - Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)" + warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" return } } From dd797d6532459198ac120ec8c14bd825864cafdc Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Mon, 13 Jun 2022 23:15:55 +0800 Subject: [PATCH 21/28] fix for some rare cases --- libexec/scoop-virustotal.ps1 | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 33c40554e2..de3af7faef 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -166,11 +166,16 @@ Function Get-VirusTotalResultByUrl ($url, $app) { $result = $response.Content $id = json_path $result '$.data.id' $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null + $last_analysis_date = json_path $result '$.data.attributes.last_analysis_date' 6>$null $url_report_url = "https://www.virustotal.com/gui/url/$id" info "$app`: Url report found." if (!$hash) { - info "$app`: Related file report not found." - warn "$app`: Manual file upload is required (instead of url submission)." + if (!$last_analysis_date) { + info "$app`: Analysis in progress." + } else { + info "$app`: Related file report not found." + warn "$app`: Manual file upload is required (instead of url submission)." + } [PSCustomObject] @{ 'App.Name' = $app 'App.Hash' = $null @@ -285,6 +290,7 @@ $reports = $apps | ForEach-Object { } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION if ($_.Exception.Response.StatusCode -eq 404) { + $file_report_not_found = $true warn "$app`: File report not found. Will search by url instead." } else { if ($_.Exception.Response.StatusCode -in 204, 429) { @@ -298,7 +304,9 @@ $reports = $apps | ForEach-Object { try { $url_report = Get-VirusTotalResultByUrl $url $app $hash = $url_report.'App.Hash' - if ($hash) { + if ($hash -and ($file_report_not_found -eq $true)) { + error "$app`: Hash not matched" + } elseif ($hash) { $file_report = Get-VirusTotalResultByHash $hash $app $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' $file_report From 08e81c768d533d863edbf9a467ccf899454afae0 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Mon, 13 Jun 2022 23:38:00 +0800 Subject: [PATCH 22/28] add '--passthru' parameter --- libexec/scoop-virustotal.ps1 | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index de3af7faef..51ca752350 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -26,6 +26,7 @@ # your virustotal_api_key. # -n, --no-depends By default, all dependencies are checked too. This flag avoids it. # -u, --no-update-scoop Don't update Scoop before checking if it's outdated +# -p, --passthru Returns report . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' @@ -33,7 +34,7 @@ . "$PSScriptRoot\..\lib\install.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\depends.ps1" # 'Get-Dependency' -$opt, $apps, $err = getopt $args 'a:snu' @('arch=', 'scan', 'no-depends', 'no-update-scoop') +$opt, $apps, $err = getopt $args 'a:snu' @('arch=', 'scan', 'no-depends', 'no-update-scoop', 'passthru') if ($err) { "scoop virustotal: $err"; exit 1 } if (!$apps) { my_usage; exit 1 } $architecture = ensure_architecture ($opt.a + $opt.arch) @@ -121,16 +122,16 @@ Function Get-VirusTotalResultByHash ($hash, $app) { $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value switch ($unsafe) { 0 { - success "$app`: $unsafe/$total" + success "$app`: $unsafe/$total, see $report_url" } 1 { - warn "$app`: $unsafe/$total" + warn "$app`: $unsafe/$total, see $report_url" } 2 { - warn "$app`: $unsafe/$total" + warn "$app`: $unsafe/$total, see $report_url" } Default { - warn "$app`: $unsafe/$total" + warn "`e[31m$app`: $unsafe/$total, see $report_url`e[0m" } } $maliciousResults = $vendorResults | @@ -328,6 +329,8 @@ $reports = $apps | ForEach-Object { } } } -$reports +if ($opt.p -or $opt.'passthru') { + $reports +} exit $exit_code From 3bdd4f51fb62255efbdb741ee2b1c1206c44531d Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Mon, 13 Jun 2022 23:55:59 +0800 Subject: [PATCH 23/28] add '-p' parameter --- libexec/scoop-virustotal.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 51ca752350..2ded346698 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -34,7 +34,7 @@ . "$PSScriptRoot\..\lib\install.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\depends.ps1" # 'Get-Dependency' -$opt, $apps, $err = getopt $args 'a:snu' @('arch=', 'scan', 'no-depends', 'no-update-scoop', 'passthru') +$opt, $apps, $err = getopt $args 'a:snup' @('arch=', 'scan', 'no-depends', 'no-update-scoop', 'passthru') if ($err) { "scoop virustotal: $err"; exit 1 } if (!$apps) { my_usage; exit 1 } $architecture = ensure_architecture ($opt.a + $opt.arch) From c60558d20124e07013a70ca47168ac6fbf85e675 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Tue, 14 Jun 2022 22:48:43 +0800 Subject: [PATCH 24/28] fix special cases --- libexec/scoop-virustotal.ps1 | 92 ++++++++++++++++++++++++++++-------- 1 file changed, 73 insertions(+), 19 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 2ded346698..dc69a37282 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -26,7 +26,7 @@ # your virustotal_api_key. # -n, --no-depends By default, all dependencies are checked too. This flag avoids it. # -u, --no-update-scoop Don't update Scoop before checking if it's outdated -# -p, --passthru Returns report +# -p, --passthru Returns reports as objects . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' @@ -92,7 +92,7 @@ Function Get-RemoteFileSize ($url) { $response.Headers.'Content-Length' | ForEach-Object { [System.Convert]::ToInt32($_) } } -Function Get-VirusTotalResultByHash ($hash, $app) { +Function Get-VirusTotalResultByHash ($hash, $url, $app) { $hash = $hash.ToLower() $api_url = "https://www.virustotal.com/api/v3/files/$hash" $headers = @{} @@ -108,15 +108,19 @@ Function Get-VirusTotalResultByHash ($hash, $app) { [int]$unsafe = $malicious + $suspicious [int]$total = $unsafe + $undetected [int]$fileSize = json_path $result '$.data.attributes.size' - $report_url = "https://www.virustotal.com/gui/file/$hash" + $report_hash = json_path $result '$.data.attributes.sha256' + $report_url = "https://www.virustotal.com/gui/file/$report_hash" if ($total -eq 0) { info "$app`: Analysis in progress." [PSCustomObject] @{ - 'App.Name' = $app - 'App.Hash' = $hash - 'App.Size' = filesize $fileSize - 'FileReport.Url' = $report_url - 'UrlReport.Url' = $null + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $hash + 'App.HashType' = $null + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $report_url + 'FileReport.Hash' = $report_hash + 'UrlReport.Url' = $null } } else { $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value @@ -142,9 +146,12 @@ Function Get-VirusTotalResultByHash ($hash, $app) { Select-Object -ExpandProperty engine_name [PSCustomObject] @{ 'App.Name' = $app + 'App.Url' = $url 'App.Hash' = $hash + 'App.HashType' = $null 'App.Size' = filesize $fileSize 'FileReport.Url' = $report_url + 'FileReport.Hash' = $report_hash 'FileReport.Malicious' = if ($maliciousResults) { $maliciousResults } else { 0 } 'FileReport.Suspicious' = if ($suspiciousResults) { $suspiciousResults } else { 0 } 'FileReport.Timeout' = $timeout @@ -179,17 +186,23 @@ Function Get-VirusTotalResultByUrl ($url, $app) { } [PSCustomObject] @{ 'App.Name' = $app + 'App.Url' = $url 'App.Hash' = $null + 'App.HashType' = $null 'FileReport.Url' = $null 'UrlReport.Url' = $url_report_url + 'UrlReport.Hash' = $null } } else { info "$app`: Related file report found." [PSCustomObject] @{ 'App.Name' = $app - 'App.Hash' = $hash + 'App.Url' = $url + 'App.Hash' = $null + 'App.HashType' = $null 'FileReport.Url' = $null 'UrlReport.Url' = $url_report_url + 'UrlReport.Hash' = $hash } } } @@ -232,6 +245,7 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { info "$app`: Analysis in progress." [PSCustomObject] @{ 'App.Name' = $app + 'App.Url' = $url 'App.Size' = filesize $fileSize 'FileReport.Url' = $null 'UrlReport.Url' = $url_report_url @@ -266,23 +280,32 @@ $reports = $apps | ForEach-Object { return } + [int]$index = 0 $urls = script:url $manifest $architecture $urls | ForEach-Object { $url = $_ + $index++ + if ($urls.GetType().IsArray) { + info "$app`: url $index" + } $hash = hash_for_url $manifest $url $architecture try { $isHashUnsupported = $false if ($hash -match '(?[^:]+):(?.*)') { + $algo = $matches.algo $hash = $matches.hash - if ($matches.algo -notmatch '(md5|sha1|sha256)') { + if ($matches.algo -inotin 'md5', 'sha1', 'sha256') { $hash = $null $isHashUnsupported = $true warn "$app`: Unsupported hash $($matches.algo). Will search by url instead." } + } elseif ($hash) { + $algo = 'sha256' } if ($hash) { - $file_report = Get-VirusTotalResultByHash $hash $app + $file_report = Get-VirusTotalResultByHash $hash $url $app + $file_report.'App.HashType' = $algo $file_report return } elseif (!$isHashUnsupported) { @@ -304,21 +327,52 @@ $reports = $apps | ForEach-Object { try { $url_report = Get-VirusTotalResultByUrl $url $app - $hash = $url_report.'App.Hash' - if ($hash -and ($file_report_not_found -eq $true)) { - error "$app`: Hash not matched" - } elseif ($hash) { - $file_report = Get-VirusTotalResultByHash $hash $app - $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' - $file_report - } else { + $url_report.'App.Hash' = $hash + $url_report.'App.HashType' = $algo + if ($url_report.'UrlReport.Hash' -and ($file_report_not_found -eq $true) -and $hash) { + if ($algo -eq 'sha256') { + if ($url_report.'UrlReport.Hash' -eq $hash) { + warn "$app`: Manual file upload is required (instead of url submission) for $url" + } else { + error "$app`: Hash not matched for $url" + } + } else { + error "$app`: Hash not matched or manual file upload is required (instead of url submission) for $url" + } $url_report + return + } + if (!$url_report.'UrlReport.Hash') { + $url_report + return } } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION if ($_.Exception.Response.StatusCode -eq 404) { warn "$app`: Url report not found. Will submit $url" Submit-ToVirusTotal $url $app ($opt.scan -or $opt.s) + return + } else { + if ($_.Exception.Response.StatusCode -in 204, 429) { + abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code + } + warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" + return + } + } + + try { + $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app + $file_report.'App.Hash' = $hash + $file_report.'App.HashType' = $algo + $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' + $file_report + warn "$app`: Unable to check hash match for $url" + } catch [Exception] { + $exit_code = $exit_code -bor $_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + warn "$app`: File report not found for unknown reason. Manual file upload is required (instead of url submission)." + $url_report } else { if ($_.Exception.Response.StatusCode -in 204, 429) { abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code From 92e481c6e037ee26c1a150ef4764050ee4644521 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <32133670+ClassicDarkChocolate@users.noreply.github.com> Date: Wed, 15 Jun 2022 00:05:55 +0800 Subject: [PATCH 25/28] fix help message Co-authored-by: Hsiao-nan Cheung --- libexec/scoop-virustotal.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index dc69a37282..7f68a4748d 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -26,7 +26,7 @@ # your virustotal_api_key. # -n, --no-depends By default, all dependencies are checked too. This flag avoids it. # -u, --no-update-scoop Don't update Scoop before checking if it's outdated -# -p, --passthru Returns reports as objects +# -p, --passthru Return reports as objects . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' From 1d05ea3edda655f2c4eeed569bc247208d26cd56 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <32133670+ClassicDarkChocolate@users.noreply.github.com> Date: Wed, 15 Jun 2022 00:06:13 +0800 Subject: [PATCH 26/28] add API Key hint Co-authored-by: Hsiao-nan Cheung --- libexec/scoop-virustotal.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 7f68a4748d..37da80ce52 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -68,8 +68,9 @@ $exit_code = 0 # Global API key: $api_key = get_config virustotal_api_key if (!$api_key) { - abort ("VirusTotal API key is not configured`n`n" + - "`tscoop config virustotal_api_key ") $_ERR_NO_API_KEY + abort ("VirusTotal API key is not configured`n" + + " You could get one from https://www.virustotal.com/gui/my-apikey and set with`n" + + " scoop config virustotal_api_key ") $_ERR_NO_API_KEY } # Global flag to explain only once about sleep between requests From 0d886792f808312dd520c9bc59c989933505c73f Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Wed, 15 Jun 2022 00:12:20 +0800 Subject: [PATCH 27/28] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dbee6ed5c..9289fac192 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Features - **core:** Add `Get-Encoding` function to fix missing webclient encoding ([#4956](https://github.com/ScoopInstaller/Scoop/issues/4956)) +- **scoop-virustotal:** migrate to virustotal api v3, improve flows, and ouput powershell objects ([#4983](https://github.com/ScoopInstaller/Scoop/pull/4983)) ### Bug Fixes From 6ced2c30187e676d892a26a31ccfa8b0f170a658 Mon Sep 17 00:00:00 2001 From: ClassicDarkChocolate <> Date: Wed, 15 Jun 2022 00:14:10 +0800 Subject: [PATCH 28/28] fix case --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9289fac192..1d1b00c615 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### Features - **core:** Add `Get-Encoding` function to fix missing webclient encoding ([#4956](https://github.com/ScoopInstaller/Scoop/issues/4956)) -- **scoop-virustotal:** migrate to virustotal api v3, improve flows, and ouput powershell objects ([#4983](https://github.com/ScoopInstaller/Scoop/pull/4983)) +- **scoop-virustotal:** Migrate to virustotal api v3, improve flows, and ouput powershell objects ([#4983](https://github.com/ScoopInstaller/Scoop/pull/4983)) ### Bug Fixes