Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(autoupdate): Fix file hash extraction #5295

Merged
merged 2 commits into from
Dec 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@

- **scoop-update:** Add support for parallel syncing buckets in PowerShell 7 and improve output ([#5122](https://github.com/ScoopInstaller/Scoop/issues/5122))

### Bug Fixes

- **autoupdate:** Fix file hash extraction ([#5295](https://github.com/ScoopInstaller/Scoop/issues/5295))

### Code Refactoring

- **git:** Use Invoke-Git() with direct path to git.exe to prevent spawning shim subprocesses ([#5122](https://github.com/ScoopInstaller/Scoop/issues/5122))
Expand Down
139 changes: 72 additions & 67 deletions lib/autoupdate.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ function find_hash_in_rdf([String] $url, [String] $basename) {
$wc.Headers.Add('User-Agent', (Get-UserAgent))
$data = $wc.DownloadData($url)
[xml]$xml = (Get-Encoding($wc)).GetString($data)
} catch [system.net.webexception] {
write-host -f darkred $_
write-host -f darkred "URL $url is not valid"
} catch [System.Net.WebException] {
Write-Host $_ -ForegroundColor DarkRed
Write-Host "URL $url is not valid" -ForegroundColor DarkRed
return $null
}

Expand All @@ -24,12 +24,12 @@ function find_hash_in_textfile([String] $url, [Hashtable] $substitutions, [Strin
$hashfile = $null

$templates = @{
'$md5' = '([a-fA-F0-9]{32})';
'$sha1' = '([a-fA-F0-9]{40})';
'$sha256' = '([a-fA-F0-9]{64})';
'$sha512' = '([a-fA-F0-9]{128})';
'$checksum' = '([a-fA-F0-9]{32,128})';
'$base64' = '([a-zA-Z0-9+\/=]{24,88})';
'$md5' = '([a-fA-F0-9]{32})'
'$sha1' = '([a-fA-F0-9]{40})'
'$sha256' = '([a-fA-F0-9]{64})'
'$sha512' = '([a-fA-F0-9]{128})'
'$checksum' = '([a-fA-F0-9]{32,128})'
'$base64' = '([a-zA-Z0-9+\/=]{24,88})'
}

try {
Expand All @@ -39,8 +39,8 @@ function find_hash_in_textfile([String] $url, [Hashtable] $substitutions, [Strin
$data = $wc.DownloadData($url)
$hashfile = (Get-Encoding($wc)).GetString($data)
} catch [system.net.webexception] {
write-host -f darkred $_
write-host -f darkred "URL $url is not valid"
Write-Host $_ -ForegroundColor DarkRed
Write-Host "URL $url is not valid" -ForegroundColor DarkRed
return
}

Expand All @@ -50,15 +50,15 @@ function find_hash_in_textfile([String] $url, [Hashtable] $substitutions, [Strin

$regex = substitute $regex $templates $false
$regex = substitute $regex $substitutions $true
debug $regex
if ($hashfile -match $regex) {
$hash = $matches[1] -replace '\s',''
debug $regex
$hash = $matches[1] -replace '\s', ''
}

# convert base64 encoded hash values
if ($hash -match '^(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=|[A-Za-z0-9+\/]{4})$') {
$base64 = $matches[0]
if(!($hash -match '^[a-fA-F0-9]+$') -and $hash.length -notin @(32, 40, 64, 128)) {
if (!($hash -match '^[a-fA-F0-9]+$') -and $hash.Length -notin @(32, 40, 64, 128)) {
try {
$hash = ([System.Convert]::FromBase64String($base64) | ForEach-Object { $_.ToString('x2') }) -join ''
} catch {
Expand All @@ -69,13 +69,15 @@ function find_hash_in_textfile([String] $url, [Hashtable] $substitutions, [Strin

# find hash with filename in $hashfile
if ($hash.Length -eq 0) {
$filenameRegex = "([a-fA-F0-9]{32,128})[\x20\t]+.*`$basename(?:[\x20\t]+\d+)?"
$filenameRegex = "([a-fA-F0-9]{32,128})[\x20\t]+.*`$basename(?:\s|$)|`$basename[\x20\t]+.*?([a-fA-F0-9]{32,128})"
$filenameRegex = substitute $filenameRegex $substitutions $true
if ($hashfile -match $filenameRegex) {
debug $filenameRegex
$hash = $matches[1]
}
$metalinkRegex = "<hash[^>]+>([a-fA-F0-9]{64})"
$metalinkRegex = '<hash[^>]+>([a-fA-F0-9]{64})'
if ($hashfile -match $metalinkRegex) {
debug $metalinkRegex
$hash = $matches[1]
}
}
Expand All @@ -92,13 +94,14 @@ function find_hash_in_json([String] $url, [Hashtable] $substitutions, [String] $
$wc.Headers.Add('User-Agent', (Get-UserAgent))
$data = $wc.DownloadData($url)
$json = (Get-Encoding($wc)).GetString($data)
} catch [system.net.webexception] {
write-host -f darkred $_
write-host -f darkred "URL $url is not valid"
} catch [System.Net.WebException] {
Write-Host $_ -ForegroundColor DarkRed
Write-Host "URL $url is not valid" -ForegroundColor DarkRed
return
}
debug $jsonpath
$hash = json_path $json $jsonpath $substitutions
if(!$hash) {
if (!$hash) {
$hash = json_path_legacy $json $jsonpath $substitutions
}
return format_hash $hash
Expand All @@ -114,8 +117,8 @@ function find_hash_in_xml([String] $url, [Hashtable] $substitutions, [String] $x
$data = $wc.DownloadData($url)
$xml = [xml]((Get-Encoding($wc)).GetString($data))
} catch [system.net.webexception] {
write-host -f darkred $_
write-host -f darkred "URL $url is not valid"
Write-Host $_ -ForegroundColor DarkRed
Write-Host "URL $url is not valid" -ForegroundColor DarkRed
return
}

Expand All @@ -125,13 +128,15 @@ function find_hash_in_xml([String] $url, [Hashtable] $substitutions, [String] $x
}

# Find all `significant namespace declarations` from the XML file
$nsList = $xml.SelectNodes("//namespace::*[not(. = ../../namespace::*)]")
$nsList = $xml.SelectNodes('//namespace::*[not(. = ../../namespace::*)]')
# Then add them into the NamespaceManager
$nsmgr = New-Object System.Xml.XmlNamespaceManager($xml.NameTable)
$nsList | ForEach-Object {
$nsmgr.AddNamespace($_.LocalName, $_.Value)
}

debug $xpath
debug $nsmgr
# Getting hash from XML, using XPath
$hash = $xml.SelectSingleNode($xpath, $nsmgr).'#text'
return format_hash $hash
Expand All @@ -148,16 +153,16 @@ function find_hash_in_headers([String] $url) {
$req.Timeout = 2000
$req.Method = 'HEAD'
$res = $req.GetResponse()
if(([int]$res.StatusCode -ge 300) -and ([int]$res.StatusCode -lt 400)) {
if($res.Headers['Digest'] -match 'SHA-256=([^,]+)' -or $res.Headers['Digest'] -match 'SHA=([^,]+)' -or $res.Headers['Digest'] -match 'MD5=([^,]+)') {
if (([int]$res.StatusCode -ge 300) -and ([int]$res.StatusCode -lt 400)) {
if ($res.Headers['Digest'] -match 'SHA-256=([^,]+)' -or $res.Headers['Digest'] -match 'SHA=([^,]+)' -or $res.Headers['Digest'] -match 'MD5=([^,]+)') {
$hash = ([System.Convert]::FromBase64String($matches[1]) | ForEach-Object { $_.ToString('x2') }) -join ''
debug $hash
}
}
$res.Close()
} catch [system.net.webexception] {
write-host -f darkred $_
write-host -f darkred "URL $url is not valid"
} catch [System.Net.WebException] {
Write-Host $_ -ForegroundColor DarkRed
Write-Host "URL $url is not valid" -ForegroundColor DarkRed
return
}

Expand All @@ -182,10 +187,10 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u
$hashfile_url = substitute $config.url $substitutions
debug $hashfile_url
if ($hashfile_url) {
write-host -f DarkYellow 'Searching hash for ' -NoNewline
write-host -f Green $basename -NoNewline
write-host -f DarkYellow ' in ' -NoNewline
write-host -f Green $hashfile_url
Write-Host 'Searching hash for ' -ForegroundColor DarkYellow -NoNewline
Write-Host $basename -ForegroundColor Green -NoNewline
Write-Host ' in ' -ForegroundColor DarkYellow -NoNewline
Write-Host $hashfile_url -ForegroundColor Green
}

if ($hashmode.Length -eq 0 -and $config.url.Length -ne 0) {
Expand Down Expand Up @@ -215,11 +220,11 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u
$hashmode = 'xpath'
}

if (!$hashfile_url -and $url -match "^(?:.*fosshub.com\/).*(?:\/|\?dwl=)(?<filename>.*)$") {
if (!$hashfile_url -and $url -match '^(?:.*fosshub.com\/).*(?:\/|\?dwl=)(?<filename>.*)$') {
$hashmode = 'fosshub'
}

if (!$hashfile_url -and $url -match "(?:downloads\.)?sourceforge.net\/projects?\/(?<project>[^\/]+)\/(?:files\/)?(?<file>.*)") {
if (!$hashfile_url -and $url -match '(?:downloads\.)?sourceforge.net\/projects?\/(?<project>[^\/]+)\/(?:files\/)?(?<file>.*)') {
$hashmode = 'sourceforge'
}

Expand All @@ -243,7 +248,7 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u
}
}
'fosshub' {
$hash = find_hash_in_textfile $url $substitutions ($Matches.filename+'.*?"sha256":"([a-fA-F0-9]{64})"')
$hash = find_hash_in_textfile $url $substitutions ($matches.filename + '.*?"sha256":"([a-fA-F0-9]{64})"')
}
'sourceforge' {
# change the URL because downloads.sourceforge.net doesn't have checksums
Expand All @@ -254,29 +259,29 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u

if ($hash) {
# got one!
write-host -f DarkYellow 'Found: ' -NoNewline
write-host -f Green $hash -NoNewline
write-host -f DarkYellow ' using ' -NoNewline
write-host -f Green "$((Get-Culture).TextInfo.ToTitleCase($hashmode)) Mode"
Write-Host 'Found: ' -ForegroundColor DarkYellow -NoNewline
Write-Host $hash -ForegroundColor Green -NoNewline
Write-Host ' using ' -ForegroundColor DarkYellow -NoNewline
Write-Host "$((Get-Culture).TextInfo.ToTitleCase($hashmode)) Mode" -ForegroundColor Green
return $hash
} elseif ($hashfile_url) {
write-host -f DarkYellow "Could not find hash in $hashfile_url"
Write-Host -f DarkYellow "Could not find hash in $hashfile_url"
}

write-host -f DarkYellow 'Downloading ' -NoNewline
write-host -f Green $basename -NoNewline
write-host -f DarkYellow ' to compute hashes!'
Write-Host 'Downloading ' -ForegroundColor DarkYellow -NoNewline
Write-Host $basename -ForegroundColor Green -NoNewline
Write-Host ' to compute hashes!' -ForegroundColor DarkYellow
try {
Invoke-CachedDownload $app $version $url $null $null $true
} catch [system.net.webexception] {
write-host -f darkred $_
write-host -f darkred "URL $url is not valid"
Write-Host $_ -ForegroundColor DarkRed
Write-Host "URL $url is not valid" -ForegroundColor DarkRed
return $null
}
$file = fullpath (cache_path $app $version $url)
$hash = (Get-FileHash -Path $file -Algorithm SHA256).Hash.ToLower()
write-host -f DarkYellow 'Computed hash: ' -NoNewline
write-host -f Green $hash
Write-Host 'Computed hash: ' -ForegroundColor DarkYellow -NoNewline
Write-Host $hash -ForegroundColor Green
return $hash
}

Expand Down Expand Up @@ -346,7 +351,7 @@ function Update-ManifestProperty {
$newValue = substitute $autoupdateProperty $Substitutions
if (($autoupdateProperty.GetType().Name -eq 'Object[]') -and ($autoupdateProperty.Length -eq 1)) {
# Make sure it's an array
$newValue = ,$newValue
$newValue = , $newValue
}
$Manifest.$currentProperty, $hasPropertyChanged = PropertyHelper -Property $Manifest.$currentProperty -Value $newValue
$hasManifestChanged = $hasManifestChanged -or $hasPropertyChanged
Expand All @@ -359,7 +364,7 @@ function Update-ManifestProperty {
$newValue = substitute $autoupdateProperty $Substitutions
if (($autoupdateProperty.GetType().Name -eq 'Object[]') -and ($autoupdateProperty.Length -eq 1)) {
# Make sure it's an array
$newValue = ,$newValue
$newValue = , $newValue
}
$Manifest.architecture.$arch.$currentProperty, $hasPropertyChanged = PropertyHelper -Property $Manifest.architecture.$arch.$currentProperty -Value $newValue
$hasManifestChanged = $hasManifestChanged -or $hasPropertyChanged
Expand Down Expand Up @@ -388,25 +393,25 @@ function Get-VersionSubstitution {
$firstPart = $Version.Split('-') | Select-Object -First 1
$lastPart = $Version.Split('-') | Select-Object -Last 1
$versionVariables = @{
'$version' = $Version;
'$dotVersion' = ($Version -replace '[._-]', '.');
'$underscoreVersion' = ($Version -replace '[._-]', '_');
'$dashVersion' = ($Version -replace '[._-]', '-');
'$cleanVersion' = ($Version -replace '[._-]', '');
'$majorVersion' = $firstPart.Split('.') | Select-Object -First 1;
'$minorVersion' = $firstPart.Split('.') | Select-Object -Skip 1 -First 1;
'$patchVersion' = $firstPart.Split('.') | Select-Object -Skip 2 -First 1;
'$buildVersion' = $firstPart.Split('.') | Select-Object -Skip 3 -First 1;
'$preReleaseVersion' = $lastPart;
}
if($Version -match "(?<head>\d+\.\d+(?:\.\d+)?)(?<tail>.*)") {
$versionVariables.Set_Item('$matchHead', $Matches['head'])
$versionVariables.Set_Item('$matchTail', $Matches['tail'])
}
if($CustomMatches) {
'$version' = $Version
'$dotVersion' = ($Version -replace '[._-]', '.')
'$underscoreVersion' = ($Version -replace '[._-]', '_')
'$dashVersion' = ($Version -replace '[._-]', '-')
'$cleanVersion' = ($Version -replace '[._-]', '')
'$majorVersion' = $firstPart.Split('.') | Select-Object -First 1
'$minorVersion' = $firstPart.Split('.') | Select-Object -Skip 1 -First 1
'$patchVersion' = $firstPart.Split('.') | Select-Object -Skip 2 -First 1
'$buildVersion' = $firstPart.Split('.') | Select-Object -Skip 3 -First 1
'$preReleaseVersion' = $lastPart
}
if ($Version -match '(?<head>\d+\.\d+(?:\.\d+)?)(?<tail>.*)') {
$versionVariables.Add('$matchHead', $Matches['head'])
$versionVariables.Add('$matchTail', $Matches['tail'])
}
if ($CustomMatches) {
$CustomMatches.GetEnumerator() | ForEach-Object {
if($_.Name -ne "0") {
$versionVariables.Set_Item('$match' + (Get-Culture).TextInfo.ToTitleCase($_.Name), $_.Value)
if ($_.Name -ne '0') {
$versionVariables.Add('$match' + (Get-Culture).TextInfo.ToTitleCase($_.Name), $_.Value)
}
}
}
Expand Down