From b2f8fb83391b64fa4561d681ebe22e70998ab10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8C=C3=A1bera?= Date: Thu, 19 Nov 2020 11:12:26 -0800 Subject: [PATCH 01/10] refactor(autoupdate): Preparation for array support #27 Co-Authored-By: Hsiao-nan Cheung --- bin/checkver.ps1 | 4 ++-- lib/autoupdate.ps1 | 6 +++--- lib/manifest.ps1 | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/checkver.ps1 b/bin/checkver.ps1 index dbcb75d187..8e24653789 100644 --- a/bin/checkver.ps1 +++ b/bin/checkver.ps1 @@ -203,7 +203,7 @@ function Invoke-Check { try { if ($Version -ne '') { $ver = $Version } - autoupdate $appName $Dir $json $ver $matchesHashtable + Invoke-Autoupdate $appName $Dir $json $ver $matchesHashtable } catch { Write-UserMessage -Message $_.Exception.Message -Err } @@ -223,7 +223,7 @@ Get-ChildItem $Dir "$Search.*" -File | ForEach-Object { foreach ($q in $Queue) { $name, $json = $q - $substitutions = get_version_substitutions $json.version + $substitutions = Get-VersionSubstitution $json.version $wc = New-Object System.Net.Webclient $ua = $json.checkver.useragent diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index 7c37d61405..ea1432be77 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -314,7 +314,7 @@ function update_manifest_prop([String] $prop, $json, [Hashtable] $substitutions) } } -function get_version_substitutions([String] $version, [Hashtable] $customMatches) { +function Get-VersionSubstitution ([String] $version, [Hashtable] $customMatches = @{ }) { $firstPart = $version -split '-' | Select-Object -First 1 $lastPart = $version -split '-' | Select-Object -Last 1 $versionVariables = @{ @@ -346,12 +346,12 @@ function get_version_substitutions([String] $version, [Hashtable] $customMatches return $versionVariables } -function autoupdate([String] $app, $dir, $json, [String] $version, [Hashtable] $MatchesHashtable) { +function Invoke-Autoupdate ([String] $app, $dir, $json, [String] $version, [Hashtable] $MatchesHashtable) { Write-UserMessage -Message "Autoupdating $app" -Color DarkCyan $has_changes = $false $has_errors = $false [bool] $valid = $true - $substitutions = get_version_substitutions $version $MatchesHashtable + $substitutions = Get-VersionSubstitution $version $MatchesHashtable if ($json.url) { # Create new url diff --git a/lib/manifest.ps1 b/lib/manifest.ps1 index 24caea0377..3721266586 100644 --- a/lib/manifest.ps1 +++ b/lib/manifest.ps1 @@ -140,7 +140,7 @@ function generate_user_manifest($app, $bucket, $version) { $path = usermanifestsdir | ensure try { - autoupdate $app "$path" $manifest $version $(@{ }) + Invoke-Autoupdate $app "$path" $manifest $version $(@{ }) return (usermanifest $app | Resolve-Path).Path } catch { From 61558cf263abd868f9196ad485e4469ffec3bc9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8C=C3=A1bera?= Date: Sat, 28 Nov 2020 02:23:06 -0800 Subject: [PATCH 02/10] backbone --- lib/core.ps1 | 65 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/lib/core.ps1 b/lib/core.ps1 index 1b1aeffc0d..806a67dfad 100644 --- a/lib/core.ps1 +++ b/lib/core.ps1 @@ -825,22 +825,65 @@ function is_scoop_outdated() { return $res } -function substitute($entity, [Hashtable] $params, [Bool]$regexEscape = $false) { - if ($entity -is [Array]) { - return $entity | ForEach-Object { substitute $_ $params $regexEscape } - } elseif ($entity -is [String]) { - $params.GetEnumerator() | ForEach-Object { - $value = if (($regexEscape -eq $false) -or ($null -eq $_.Value)) { $_.Value } else { [Regex]::Escape($_.Value) } - $curly = '${' + $_.Name.TrimStart('$') + '}' - - $entity = $entity.Replace($curly, $value) - $entity = $entity.Replace($_.Name, $value) +function Invoke-VariableSubstitution { + <# + .SYNOPSIS + Substitute (find and replace) provided parameters in provided entity. + .PARAMETER Entity + Specifies the entity to be substituted (searched in). + .PARAMETER Parameters + Specifies the hashtable providing name and value pairs for "find and replace". + Hashtable keys should start with $ (dollar sign). Curly bracket variable syntax will be substituted automatically. + .PARAMETER EscapeRegularExpression + Specifies to escape regular expressions before replacing values. + #> + [CmdletBinding()] + param( + [AllowEmptyCollection()] + [AllowNull()] + $Entity, + [Parameter(Mandatory)] + [HashTable] $Parameters, + [Switch] $EscapeRegularExpression + ) + + process { + $EscapeRegularExpression | Out-Null # PowerShell/PSScriptAnalyzer#1472 + $newEntity = $Entity + + if ($null -ne $newEntity) { + switch ($newEntity.GetType().Name) { + 'String' { + $Parameters.GetEnumerator() | ForEach-Object { + $value = if (($EscapeRegularExpression -eq $false) -or ($null -eq $_.Value)) { $_.Value } else { [Regex]::Escape($_.Value) } + $curly = '${' + $_.Name.TrimStart('$') + '}' + + $newEntity = $newEntity.Replace($curly, $value) + $newEntity = $newEntity.Replace($_.Name, $value) + } + } + 'Object[]' { + $newEntity = $newEntity | ForEach-Object { Invoke-VariableSubstitution -Entity $_ -Parameters $params -EscapeRegularExpression:$regexEscape } + } + 'PSCustomObject' { + $newentity.PSObject.Properties | ForEach-Object { $_.Value = Invoke-VariableSubstitution -Entity $_ -Parameters $params -EscapeRegularExpression:$regexEscape } + } + default { + # This is not needed, but to cover all possible use cases explicitly + $newEntity = $newEntity + } + } } - return $entity + return $newEntity } } +# TODO: Deprecate +function substitute($entity, [Hashtable] $params, [Bool]$regexEscape = $false) { + return Invoke-VariableSubstitution -Entity $entity -Parameters $params -EscapeRegularExpression:$regexEscape +} + function format_hash([String] $hash) { # Convert base64 encoded hash values if ($hash -match '^(?:[A-Za-z\d+\/]{4})*(?:[A-Za-z\d+\/]{2}==|[A-Za-z\d+\/]{3}=|[A-Za-z\d+\/]{4})$') { From e3ba82edf435d35c71e0c0b3327c9ad0cce62f95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8C=C3=A1bera?= Date: Sat, 28 Nov 2020 03:40:01 -0800 Subject: [PATCH 03/10] call changes --- bin/checkver.ps1 | 4 ++-- lib/autoupdate.ps1 | 18 +++++++++--------- lib/install.ps1 | 4 ++-- lib/json.ps1 | 4 ++-- lib/shortcuts.ps1 | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/bin/checkver.ps1 b/bin/checkver.ps1 index 8e24653789..afe82a93b0 100644 --- a/bin/checkver.ps1 +++ b/bin/checkver.ps1 @@ -227,7 +227,7 @@ foreach ($q in $Queue) { $wc = New-Object System.Net.Webclient $ua = $json.checkver.useragent - $ua = if ($ua) { substitute $ua $substitutions } else { Get-UserAgent } + $ua = if ($ua) { Invoke-VariableSubstitution -Entity $ua -Parameters $substitutions } else { Get-UserAgent } $wc.Headers.Add('User-Agent', $ua) Register-ObjectEvent $wc DownloadStringCompleted -ErrorAction Stop | Out-Null @@ -270,7 +270,7 @@ foreach ($q in $Queue) { $regex = if ($json.checkver -is [System.String]) { $json.checkver } else { $UNIVERSAL_REGEX } } - $url = substitute $url $substitutions + $url = Invoke-VariableSubstitution -Entity $url -Parameters $substitutions $state = New-Object PSObject @{ 'app' = (strip_ext $name) diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index ea1432be77..efee88e4f1 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -49,8 +49,8 @@ function find_hash_in_textfile([String] $url, [Hashtable] $substitutions, [Strin if ($regex.Length -eq 0) { $regex = '^([a-fA-F\d]+)$' } - $regex = substitute $regex $templates $false - $regex = substitute $regex $substitutions $true + $regex = Invoke-VariableSubstitution -Entity $regex -Parameters $templates -EscapeRegularExpression:$false + $regex = Invoke-VariableSubstitution -Entity $regex -Parameters $substitutions -EscapeRegularExpression:$true debug $regex @@ -59,7 +59,7 @@ function find_hash_in_textfile([String] $url, [Hashtable] $substitutions, [Strin # Find hash with filename in $hashfile if ($hash.Length -eq 0) { $filenameRegex = "([a-fA-F\d]{32,128})[\x20\t]+.*`$basename(?:[\x20\t]+\d+)?" - $filenameRegex = substitute $filenameRegex $substitutions $true + $filenameRegex = Invoke-VariableSubstitution -Entity $filenameRegex -Parameters $substitutions -EscapeRegularExpression:$true if ($hashfile -match $filenameRegex) { $hash = $Matches[1] } @@ -111,7 +111,7 @@ function find_hash_in_xml([String] $url, [Hashtable] $substitutions, [String] $x $xml = [xml] $xml # Replace placeholders - if ($substitutions) { $xpath = substitute $xpath $substitutions } + if ($substitutions) { $xpath = Invoke-VariableSubstitution -Entity $xpath -Parameters $substitutions } # Find all `significant namespace declarations` from the XML file $nsList = $xml.SelectNodes('//namespace::*[not(. = ../../namespace::*)]') @@ -168,7 +168,7 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u debug $substitutions - $hashfile_url = substitute $config.url $substitutions + $hashfile_url = Invoke-VariableSubstitution -Entity $config.url -Parameters $substitutions debug $hashfile_url @@ -300,7 +300,7 @@ function update_manifest_with_new_version($json, [String] $version, [String] $ur function update_manifest_prop([String] $prop, $json, [Hashtable] $substitutions) { # first try the global property if ($json.$prop -and $json.autoupdate.$prop) { - $json.$prop = substitute $json.autoupdate.$prop $substitutions + $json.$prop = Invoke-VariableSubstitution -Entity $json.autoupdate.$prop -Parameters $substitutions } # check if there are architecture specific variants @@ -308,7 +308,7 @@ function update_manifest_prop([String] $prop, $json, [Hashtable] $substitutions) $json.architecture | Get-Member -MemberType NoteProperty | ForEach-Object { $architecture = $_.Name if ($json.architecture.$architecture.$prop -and $json.autoupdate.architecture.$architecture.$prop) { - $json.architecture.$architecture.$prop = substitute (arch_specific $prop $json.autoupdate $architecture) $substitutions + $json.architecture.$architecture.$prop = Invoke-VariableSubstitution -Entity (arch_specific $prop $json.autoupdate $architecture) -Parameters $substitutions } } } @@ -355,7 +355,7 @@ function Invoke-Autoupdate ([String] $app, $dir, $json, [String] $version, [Hash if ($json.url) { # Create new url - $url = substitute $json.autoupdate.url $substitutions + $url = Invoke-VariableSubstitution -Entity $json.autoupdate.url -Parameters $substitutions $valid = $true if ($valid) { @@ -381,7 +381,7 @@ function Invoke-Autoupdate ([String] $app, $dir, $json, [String] $version, [Hash $architecture = $_.Name # Create new url - $url = substitute (arch_specific "url" $json.autoupdate $architecture) $substitutions + $url = Invoke-VariableSubstitution -Entity (arch_specific "url" $json.autoupdate $architecture) -Parameters $substitutions $valid = $true if ($valid) { diff --git a/lib/install.ps1 b/lib/install.ps1 index 4d5b16b52d..ecbad2b6e8 100644 --- a/lib/install.ps1 +++ b/lib/install.ps1 @@ -875,7 +875,7 @@ function create_shims($manifest, $dir, $global, $arch) { if (!$bin) { throw [ScoopException] "Shim creation fail|-Cannot shim '$target': File does not exist" } # TerminatingError thrown - shim $bin $global $name (substitute $arg @{ '$dir' = $dir; '$original_dir' = $original_dir; '$persist_dir' = $persist_dir }) + shim $bin $global $name (Invoke-VariableSubstitution -Entity $arg -Parameters @{ '$dir' = $dir; '$original_dir' = $original_dir; '$persist_dir' = $persist_dir }) } } @@ -1067,7 +1067,7 @@ function show_notes($manifest, $dir, $original_dir, $persist_dir) { Write-UserMessage -Output:$false -Message @( 'Notes' '-----' - (wraptext (substitute $manifest.notes @{ '$dir' = $dir; '$original_dir' = $original_dir; '$persist_dir' = $persist_dir })) + (wraptext (Invoke-VariableSubstitution -Entity $manifest.notes -Parameters @{ '$dir' = $dir; '$original_dir' = $original_dir; '$persist_dir' = $persist_dir })) ) } } diff --git a/lib/json.ps1 b/lib/json.ps1 index 8e9d1754cb..b6c568c539 100644 --- a/lib/json.ps1 +++ b/lib/json.ps1 @@ -89,7 +89,7 @@ function ConvertToPrettyJson { function json_path([String] $json, [String] $jsonpath, [Hashtable] $substitutions) { Add-Type -Path (Join-Path $PSScriptRoot '\..\supporting\validator\bin\Newtonsoft.Json.dll') - if ($null -ne $substitutions) { $jsonpath = substitute $jsonpath $substitutions ($jsonpath -like '*=~*') } + if ($null -ne $substitutions) { $jsonpath = Invoke-VariableSubstitution -Entity $jsonpath -Parameters $substitutions -EscapeRegularExpression:($jsonpath -like '*=~*') } try { $obj = [Newtonsoft.Json.Linq.JObject]::Parse($json) } catch [Newtonsoft.Json.JsonReaderException] { @@ -122,7 +122,7 @@ function json_path_legacy([String] $json, [String] $jsonpath, [Hashtable] $subst $el = $_ # Substitute the basename and version varibales into the jsonpath - if ($null -ne $substitutions) { $el = substitute $el $substitutions } + if ($null -ne $substitutions) { $el = Invoke-VariableSubstitution -Entity $el -Parameters $substitutions } # Skip $ if it's jsonpath format if ($el -eq '$' -and $isJsonPath) { return } diff --git a/lib/shortcuts.ps1 b/lib/shortcuts.ps1 index 3f5018d0a1..11e042669f 100644 --- a/lib/shortcuts.ps1 +++ b/lib/shortcuts.ps1 @@ -18,7 +18,7 @@ function create_startmenu_shortcuts($manifest, $dir, $global, $arch) { $icon = [System.IO.Path]::Combine($dir, $_.item(3)) $icon = New-Object System.IO.FileInfo($icon) } - $arguments = (substitute $arguments @{ '$dir' = $dir; '$original_dir' = $original_dir; '$persist_dir' = $persist_dir }) + $arguments = (Invoke-VariableSubstitution -Entity $arguments -Parameters @{ '$dir' = $dir; '$original_dir' = $original_dir; '$persist_dir' = $persist_dir }) startmenu_shortcut $target $name $arguments $icon $global } } From 0108cee459db52930193194a9cd3781c3f74974d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8C=C3=A1bera?= Date: Sat, 28 Nov 2020 04:02:08 -0800 Subject: [PATCH 04/10] small tweak --- bin/checkver.ps1 | 2 +- lib/autoupdate.ps1 | 28 +++++++++++++--------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/bin/checkver.ps1 b/bin/checkver.ps1 index afe82a93b0..9fd48d407b 100644 --- a/bin/checkver.ps1 +++ b/bin/checkver.ps1 @@ -223,7 +223,7 @@ Get-ChildItem $Dir "$Search.*" -File | ForEach-Object { foreach ($q in $Queue) { $name, $json = $q - $substitutions = Get-VersionSubstitution $json.version + $substitutions = Get-VersionSubstitution -Version $json.version $wc = New-Object System.Net.Webclient $ua = $json.checkver.useragent diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index efee88e4f1..9e5c9ec660 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -314,32 +314,30 @@ function update_manifest_prop([String] $prop, $json, [Hashtable] $substitutions) } } -function Get-VersionSubstitution ([String] $version, [Hashtable] $customMatches = @{ }) { - $firstPart = $version -split '-' | Select-Object -First 1 - $lastPart = $version -split '-' | Select-Object -Last 1 +function Get-VersionSubstitution ([String] $Version, [Hashtable] $CustomMatches = @{ }) { + $firstPart = $Version -split '-' | Select-Object -First 1 + $lastPart = $Version -split '-' | Select-Object -Last 1 $versionVariables = @{ - '$version' = $version - '$underscoreVersion' = ($version -replace '\.', '_') - '$dashVersion' = ($version -replace '\.', '-') - '$cleanVersion' = ($version -replace '\.') + '$version' = $Version + '$underscoreVersion' = ($Version -replace '\.', '_') + '$dashVersion' = ($Version -replace '\.', '-') + '$cleanVersion' = ($Version -replace '\.') '$majorVersion' = ($firstPart -split '\.' | Select-Object -First 1) '$minorVersion' = ($firstPart -split '\.' | Select-Object -Skip 1 -First 1) '$patchVersion' = ($firstPart -split '\.' | Select-Object -Skip 2 -First 1) '$buildVersion' = ($firstPart -split '\.' | Select-Object -Skip 3 -First 1) '$preReleaseVersion' = $lastPart } - if ($version -match '(?\d+\.\d+(?:\.\d+)?)(?.*)') { + if ($Version -match '(?\d+\.\d+(?:\.\d+)?)(?.*)') { $versionVariables.Add('$matchHead', $Matches['head']) $versionVariables.Add('$headVersion', $Matches['head']) $versionVariables.Add('$matchTail', $Matches['tail']) $versionVariables.Add('$tailVersion', $Matches['tail']) } - if ($customMatches) { - $customMatches.GetEnumerator() | ForEach-Object { - if ($_.Name -ne '0') { - # .Add() cannot be used due to unskilled maintainers, who could use internal $matchHead or $matchTail variable and recive exception - $versionVariables.set_Item('$match' + (Get-Culture).TextInfo.ToTitleCase($_.Name), $_.Value) - } + if ($CustomMatches) { + $CustomMatches.GetEnumerator() | Where-Object -Property Name -NE -Value '0' | ForEach-Object { + # .Add() cannot be used due to unskilled maintainers, who could use internal $matchHead or $matchTail variable and receive exception + $versionVariables.set_Item('$match' + (Get-Culture).TextInfo.ToTitleCase($_.Name), $_.Value) } } @@ -351,7 +349,7 @@ function Invoke-Autoupdate ([String] $app, $dir, $json, [String] $version, [Hash $has_changes = $false $has_errors = $false [bool] $valid = $true - $substitutions = Get-VersionSubstitution $version $MatchesHashtable + $substitutions = Get-VersionSubstitution -Version $version -CustomMatches $MatchesHashtable if ($json.url) { # Create new url From 91bc9df29ffbd7c7ceadd5a3826a5865d4993e85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8C=C3=A1bera?= Date: Sat, 28 Nov 2020 07:10:57 -0800 Subject: [PATCH 05/10] refactor wave 2 --- lib/autoupdate.ps1 | 254 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 198 insertions(+), 56 deletions(-) diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index 9e5c9ec660..422f3861c4 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -344,83 +344,225 @@ function Get-VersionSubstitution ([String] $Version, [Hashtable] $CustomMatches return $versionVariables } -function Invoke-Autoupdate ([String] $app, $dir, $json, [String] $version, [Hashtable] $MatchesHashtable) { - Write-UserMessage -Message "Autoupdating $app" -Color DarkCyan - $has_changes = $false - $has_errors = $false - [bool] $valid = $true - $substitutions = Get-VersionSubstitution -Version $version -CustomMatches $MatchesHashtable - - if ($json.url) { - # Create new url - $url = Invoke-VariableSubstitution -Entity $json.autoupdate.url -Parameters $substitutions - $valid = $true - - if ($valid) { - # Create hash - $hash = get_hash_for_app $app $json.autoupdate.hash $version $url $substitutions - if ($null -eq $hash) { - $valid = $false - Write-UserMessage -Message 'Could not find hash!' -Color DarkRed +function Update-ManifestProperty { + [CmdletBinding()] + [OutputType([Boolean])] + param ( + [Parameter(Mandatory)] + [Alias('InputObject')] + $Manifest, + [Parameter(ValueFromPipeline)] + [String[]] $Property, + [String] $AppName, + [String] $Version, + [HashTable] $Substitutions + ) + + begin { $manifestChanged = $false } + + process { + $AppName | Out-Null # PowerShell/PSScriptAnalyzer#1472 + + foreach ($prop in $Property) { + if ($prop -eq 'hash') { + if ($Manifest.hash) { + # Substitute new value + $newURL = Invoke-VariableSubstitution -Entity $Manifest.autoupdate.url -Parameters $Substitutions + $newHash = _getHashesForUrls -AppName $AppName -Version $Version -HashExtraction $Manifest.autoupdate.hash -URL $newURL -Substitutions $Substitutions + + # Update manifest + $Manifest.hash, $propertyChanged = _updateSpecificProperty -Property $Manifest.hash -Value $newHash + $manifestChanged = $manifestChanged -or $propertyChanged + } else { + # Arch-spec + $Manifest.architecture | Get-Member -MemberType NoteProperty | ForEach-Object { + $arch = $_.Name + # Substitute new URLS + $newURL = Invoke-VariableSubstitution -Entity (arch_specific 'url' $Manifest.autoupdate $arch) -Parameters $Substitutions + # Calculate/extract hashes + $newHash = _getHashesForUrls -AppName $AppName -Version $Version -HashExtraction (arch_specific 'hash' $Manifest.autoupdate $arch) -URL $newURL -Substitutions $Substitutions + + # Update manifest + $Manifest.architecture.$arch.hash, $propertyChanged = _updateSpecificProperty -Property $Manifest.architecture.$arch.hash -Value $newHash + $manifestChanged = $manifestChanged -or $propertyChanged + } + } + # Extract and update hash property + } elseif ($Manifest.$prop -and $Manifest.autoupdate.$Prop) { + # Substitute new value + $newValue = Invoke-VariableSubstitution -Entity $Manifest.autoupdate.$prop -Parameters $Substitutions + + # Update manifest + $Manifest.$prop, $propertyChanged = _updateSpecificProperty -Property $Manifest.$prop -Value $newValue + $manifestChanged = $manifestChanged -or $propertyChanged + } elseif ($Manifest.architecture) { + # Substitute and update architecture specific property + $Manifest.architecture | Get-Member -MemberType NoteProperty | ForEach-Object { + $arch = $_.Name + if ($Manifest.architecture.$arch.$prop -and ($Manifest.autoupdate.architecture.$arch.$prop -or $Manifest.autoupdate.$prop)) { + # Substitute new value + $newValue = Invoke-VariableSubstitution -Entity (arch_specific $prop $Manifest.autoupdate $arch) -Parameters $Substitutions + + # Update manifest + $Manifest.architecture.$arch.$prop, $propertyChanged = _updateSpecificProperty -Property $Manifest.architecture.$arch.$prop -Value $newValue + $hasManifestChanged = $hasManifestChanged -or $propertyChanged + } + } } } + } - # Write changes to the json object - if ($valid) { - $has_changes = $true - update_manifest_with_new_version $json $version $url $hash - } else { - $has_errors = $true - throw "Could not update $app" + end { + if (($Version -ne '') -and ($Manifest.version -ne $Version)) { + $Manifest.version = $Version + $manifestChanged = $true } - } else { - $json.architecture | Get-Member -MemberType NoteProperty | ForEach-Object { - $valid = $true - $architecture = $_.Name - # Create new url - $url = Invoke-VariableSubstitution -Entity (arch_specific "url" $json.autoupdate $architecture) -Parameters $substitutions - $valid = $true + return $manifestChanged + } +} - if ($valid) { - # Create hash - $hash = get_hash_for_app $app (arch_specific "hash" $json.autoupdate $architecture) $version $url $substitutions - if ($null -eq $hash) { - $valid = $false - Write-UserMessage -Message 'Could not find hash!' -Color DarkRed - } - } +function Invoke-Autoupdate ([String] $app, $dir, $json, [String] $version, [Hashtable] $MatchesHashtable) { + Write-Host "Autoupdating $app" -ForegroundColor DarkCyan + $Manifest = $json + $substitutions = Get-VersionSubstitution -Version $version -CustomMatches $MatchesHashtable - # Write changes to the json object - if ($valid) { - $has_changes = $true - update_manifest_with_new_version $json $version $url $hash $architecture - } else { - $has_errors = $true - throw "Could not update $app $architecture" - } + # Get properties, which needs to be updated + $updatedProperties = @(@($Manifest.autoupdate.PSObject.Properties.Name) -ne 'architecture') + if ($Manifest.autoupdate.architecture) { + $Manifest.autoupdate.architecture.PSObject.Properties | ForEach-Object { + $updatedProperties += $_.Value.PSObject.Properties.Name } } - # Update properties - update_manifest_prop 'extract_dir' $json $substitutions + # Hashes needs to be updated if not explicitly specified + if ($updatedProperties -contains 'url') { $updatedProperties += 'hash' } + + $updatedProperties = $updatedProperties | Select-Object -Unique + debug [$updatedProperties] - # Update license - update_manifest_prop 'license' $json $substitutions + $changed = Update-ManifestProperty -Manifest $Manifest -Property $updatedProperties -AppName $app -Version $version -Substitutions $substitutions - if ($has_changes -and !$has_errors) { - # Write file + if ($changed) { Write-UserMessage -Message "Writing updated $app manifest" -Color DarkGreen $path = Join-Path $dir "$app.json" - - $json | ConvertToPrettyJson | Out-UTF8File -Path $path + $Manifest | ConvertToPrettyJson | Out-UTF8File -Path $path # Notes if ($json.autoupdate.note) { Write-UserMessage -Message '', $json.autoupdate.note -Color DarkYellow } } else { + # This if-else branch may not be in use. Write-UserMessage -Message "No updates for $app" -Color DarkGray } } + +#region Helpers +function _updateSpecificProperty { + <# + .SYNOPSIS + Helper for updating manifest's property + .DESCRIPTION + Updates manifest property (String, Array or PSCustomObject). + .PARAMETER Property + Specifies the name of property to be updated. + .PARAMETER Value + Specifies the new value of property. + Update line by line. + .OUTPUTS + System.Object[] + The first element is new value of property, the second element is change flag + #> + param ( + [ValidateNotNull()] + [Parameter(Mandatory, ValueFromPipeline)] + [Object] $Property, + [Parameter(Mandatory)] + [Object] $Value + ) + begin { + $result = $Property + $hasChanged = $false + } + + process { + # Bind value into property in case the new value is "longer" than current. + if (@($Property).Length -lt @($Value).Length) { + $result = $Value + $hasChanged = $true + } else { + switch ($Property.GetType().Name) { + 'String' { + $val = $Value -as [String] + if ($null -ne $val) { + $result = $val + $hasChanged = $true + } + } + 'Object[]' { + $val = @($Value) + for ($i = 0; $i -lt $val.Length; $i++) { + $result[$i], $itemChanged = _updateSpecificProperty -Property $Property[$i] -Value $val[$i] + $hasChanged = $hasChanged -or $itemChanged + } + } + 'PSCustomObject' { + if ($Value -is [PSObject]) { + foreach ($name in $Property.PSObject.Properties.Name) { + if ($Value.$name) { + $result.$name, $itemChanged = _updateSpecificProperty -Property $Property.$name -Value $Value.$name + $hasChanged = $hasChanged -or $itemChanged + } + } + } + } + } + } + } + + end { return $result, $hasChanged } +} + +function _getHashesForUrls { + <# + .SYNOPSIS + Helper for extracting hashes. + .DESCRIPTION + Extract or calculate hash(es) for provided URLs. + If number of hash extraction templates is less then URLs, the last template will be reused for the rest URLs. + .PARAMETER AppName + Specifies the name of the application. + .PARAMETER Version + Specifies the version of the application. + .PARAMETER HashExtraction + Specifeis the extraction method. + .PARAMETER URL + Specifes the links to updated files. + .PARAMETER Substitutions + Specifes the hashtable with substitutions. + #> + param ( + [String] $AppName, + [String] $Version, + [PSObject[]] $HashExtraction, + [String[]] $URL, + [HashTable] $Substitutions + ) + $hash = @() + for ($i = 0; $i -lt $URL.Length; $i++) { + if ($null -eq $HashExtraction) { + $extract = $null + } else { + $extract = $HashExtraction[$i], $HashExtraction[-1] | Select-Object -First 1 + } + $hash += get_hash_for_app $AppName $extract $Version $URL[$i] $Substitutions + if ($null -eq $hash[$i]) { + throw "Could not update $AppName, hash for $(url_remote_filename $URL[$i]) failed!" + } + } + + return $hash +} +#endregion Helpers From dcab35cd1123e4ed4cbf46e68bd8afbee1936bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8C=C3=A1bera?= Date: Sat, 28 Nov 2020 08:04:44 -0800 Subject: [PATCH 06/10] fix and cleanup --- lib/autoupdate.ps1 | 40 ---------------------------------------- lib/core.ps1 | 4 ++-- 2 files changed, 2 insertions(+), 42 deletions(-) diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index 422f3861c4..b5220dd925 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -274,46 +274,6 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u return $hash } -function update_manifest_with_new_version($json, [String] $version, [String] $url, [String] $hash, $architecture = $null) { - $json.version = $version - - if ($null -eq $architecture) { - if ($json.url -is [System.Array]) { - $json.url[0] = $url - $json.hash[0] = $hash - } else { - $json.url = $url - $json.hash = $hash - } - } else { - # If there are multiple urls we replace the first one - if ($json.architecture.$architecture.url -is [System.Array]) { - $json.architecture.$architecture.url[0] = $url - $json.architecture.$architecture.hash[0] = $hash - } else { - $json.architecture.$architecture.url = $url - $json.architecture.$architecture.hash = $hash - } - } -} - -function update_manifest_prop([String] $prop, $json, [Hashtable] $substitutions) { - # first try the global property - if ($json.$prop -and $json.autoupdate.$prop) { - $json.$prop = Invoke-VariableSubstitution -Entity $json.autoupdate.$prop -Parameters $substitutions - } - - # check if there are architecture specific variants - if ($json.architecture -and $json.autoupdate.architecture) { - $json.architecture | Get-Member -MemberType NoteProperty | ForEach-Object { - $architecture = $_.Name - if ($json.architecture.$architecture.$prop -and $json.autoupdate.architecture.$architecture.$prop) { - $json.architecture.$architecture.$prop = Invoke-VariableSubstitution -Entity (arch_specific $prop $json.autoupdate $architecture) -Parameters $substitutions - } - } - } -} - function Get-VersionSubstitution ([String] $Version, [Hashtable] $CustomMatches = @{ }) { $firstPart = $Version -split '-' | Select-Object -First 1 $lastPart = $Version -split '-' | Select-Object -Last 1 diff --git a/lib/core.ps1 b/lib/core.ps1 index 806a67dfad..7bbcfba5f7 100644 --- a/lib/core.ps1 +++ b/lib/core.ps1 @@ -863,10 +863,10 @@ function Invoke-VariableSubstitution { } } 'Object[]' { - $newEntity = $newEntity | ForEach-Object { Invoke-VariableSubstitution -Entity $_ -Parameters $params -EscapeRegularExpression:$regexEscape } + $newEntity = $newEntity | ForEach-Object { Invoke-VariableSubstitution -Entity $_ -Parameters $Parameters -EscapeRegularExpression:$regexEscape } } 'PSCustomObject' { - $newentity.PSObject.Properties | ForEach-Object { $_.Value = Invoke-VariableSubstitution -Entity $_ -Parameters $params -EscapeRegularExpression:$regexEscape } + $newentity.PSObject.Properties | ForEach-Object { $_.Value = Invoke-VariableSubstitution -Entity $_ -Parameters $Parameters -EscapeRegularExpression:$regexEscape } } default { # This is not needed, but to cover all possible use cases explicitly From 99ba145a14cc3f592bfa32c68150f9a2b413f29c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8C=C3=A1bera?= Date: Tue, 13 Dec 2022 15:55:27 +0000 Subject: [PATCH 07/10] Return archivation --- lib/Helpers.ps1 | 2 +- lib/autoupdate.ps1 | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/lib/Helpers.ps1 b/lib/Helpers.ps1 index 6205f609fc..7d30d13fd1 100644 --- a/lib/Helpers.ps1 +++ b/lib/Helpers.ps1 @@ -248,7 +248,7 @@ function Invoke-VariableSubstitution { } } 'Object[]' { - $newEntity = $newEntity | ForEach-Object { Invoke-VariableSubstitution -Entity $_ -Substitutes $Substitutes -EscapeRegularExpression:$regexEscape } + $newEntity = $newEntity | ForEach-Object { , (Invoke-VariableSubstitution -Entity $_ -Substitutes $Substitutes -EscapeRegularExpression:$regexEscape) } } 'PSCustomObject' { $newentity.PSObject.Properties | ForEach-Object { $_.Value = Invoke-VariableSubstitution -Entity $_ -Substitutes $Substitutes -EscapeRegularExpression:$regexEscape } diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index 7537200b8b..a3976346e1 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -385,7 +385,13 @@ function Update-ManifestProperty { # Extract and update hash property } elseif ($Manifest.$prop -and $Manifest.autoupdate.$Prop) { # Substitute new value - $newValue = Invoke-VariableSubstitution -Entity $Manifest.autoupdate.$prop -Parameters $Substitutions + $autoupdateProperty = $Manifest.autoupdate.$prop + $newValue = Invoke-VariableSubstitution -Entity $autoupdateProperty -Substitutes $Substitutions + + if (($autoupdateProperty.GetType().Name -eq 'Object[]') -and ($autoupdateProperty.Length -eq 1)) { + # Make sure it's an array + $newValue = , $newValue + } # Update manifest $Manifest.$prop, $propertyChanged = _updateSpecificProperty -Property $Manifest.$prop -Value $newValue @@ -396,7 +402,13 @@ function Update-ManifestProperty { $arch = $_.Name if ($Manifest.architecture.$arch.$prop -and ($Manifest.autoupdate.architecture.$arch.$prop -or $Manifest.autoupdate.$prop)) { # Substitute new value - $newValue = Invoke-VariableSubstitution -Entity (arch_specific $prop $Manifest.autoupdate $arch) -Parameters $Substitutions + $autoupdateProperty = @(arch_specific $prop $Manifest.autoupdate $arch) + + $newValue = Invoke-VariableSubstitution -Entity $autoupdateProperty -Substitutes $Substitutions + if (($autoupdateProperty.GetType().Name -eq 'Object[]') -and ($autoupdateProperty.Length -eq 1)) { + # Make sure it's an array + $newValue = , $newValue + } # Update manifest $Manifest.architecture.$arch.$prop, $propertyChanged = _updateSpecificProperty -Property $Manifest.architecture.$arch.$prop -Value $newValue @@ -417,8 +429,13 @@ function Update-ManifestProperty { } } -function Invoke-Autoupdate ([String] $app, $dir, $json, [String] $version, [Hashtable] $MatchesHashtable) { +function Invoke-Autoupdate ([String] $app, $dir, $json, [String] $version, [Hashtable] $MatchesHashtable, [String] $Extension = '.json', [Switch] $IgnoreArchive) { Write-Host "Autoupdating $app" -ForegroundColor DarkCyan + + # For archive purpose + $oldVersion = $json.version + $oldJson = $json | ConvertTo-Json -Depth 50 | ConvertFrom-Json # Deep clone object + $Manifest = $json $substitutions = Get-VersionSubstitution -Version $version -CustomMatches $MatchesHashtable @@ -440,6 +457,20 @@ function Invoke-Autoupdate ([String] $app, $dir, $json, [String] $version, [Hash $newManifest = $null if ($changed) { + # Archive older version + if (!$IgnoreArchive -and ($json.autoupdate.archive -and ($json.autoupdate.archive -eq $true))) { + $appOldPath = Join-Path $dir "old\$app" + $manifestOldPath = Join-Path $appOldPath "${oldVersion}${Extension}" + + Write-UserMessage -Message "Archiving manifest with version $oldVersion to $manifestOldPath" -Info + + $oldJson.PSObject.Properties.Remove('checkver') + $oldJson.PSObject.Properties.Remove('autoupdate') + + Confirm-DirectoryExistence -LiteralPath $appOldPath | Out-Null + ConvertTo-Manifest -Manifest $oldJson -File $manifestOldPath + } + # Notes if ($json.autoupdate.note) { Write-UserMessage -Message '', $json.autoupdate.note -Color 'DarkYellow' } From 16a662f57694d14f5c17bce1e99fc00a59c6b10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8C=C3=A1bera?= Date: Tue, 13 Dec 2022 16:13:46 +0000 Subject: [PATCH 08/10] Changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20eb5bd04f..2a68af3cb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [0.6.5](https://github.com/Ash258/Scoop-Core/milestone/5) +- **Autoupdate**: Properly support multiple items for specific properties - **core**: Use `[System.IO.Path]::Combine` for path joining - **Download**: Do not show progress in CI by default - **Uninstall**: Run `post_uninstall` after current unlinking @@ -13,7 +14,7 @@ - **scoop-info**: Adopt new resolve function for parameter passing - **Diagnostic**: Ignore completion check when `-noprofile` is used (cmd usage most likely) - **commands**: Short option `-a` will not produce default architecture always -- Increase command startup +- Improve command startup - Prevent MSI installations in NanoServer images - Installation will not fail if `commonstartmenu` or `startmenu` system folder is not defined - Print not supported message only if operation will be executed From bc812de1892cbed219a8467914aca74fb1683a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8C=C3=A1bera?= Date: Sat, 31 Dec 2022 22:51:58 +0000 Subject: [PATCH 09/10] small container tweaks --- .devcontainer/postAttach.sh | 11 +++++++++-- .gitignore | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.devcontainer/postAttach.sh b/.devcontainer/postAttach.sh index 24c6b3b340..dd7f0c6280 100755 --- a/.devcontainer/postAttach.sh +++ b/.devcontainer/postAttach.sh @@ -13,6 +13,7 @@ EOF USERNAME=${USERNAME:-"vscode"} ZDOTDIR=${ZDOTDIR:-$HOME} +ZSHENVFILE="$ZDOTDIR/.zshenv" SCOOP=/home/vscode/Shovel SCOOP_HOME=${SCOOP}/apps/scoop/current SCOOP_GLOBAL=/opt/Shovel @@ -26,7 +27,7 @@ if [ -f "${SHOVEL}/shims/shovel.cmd" ]; then exit 0 fi -echo "$profileString" >> "$ZDOTDIR/.zshenv" +echo "$profileString" >> "${ZSHENVFILE}" mkdir -p "${ZDOTDIR}/.config/scoop" sudo mkdir -p "${ZDOTDIR}/Shovel/shims" "${SHOVEL_GLOBAL}/shims" @@ -38,8 +39,14 @@ wget --quiet --output-document="${SHOVEL}/shims/shovel.ps1" 'https://raw.githubu wget --quiet --output-document="${SHOVEL}/shims/shovel.cmd" 'https://raw.githubusercontent.com/shovel-org/Dockers/main/support/shovel.cmd' chmod +x "${SHOVEL}/shims/shovel"* -pwsh --version || sudo chmod +x /usr/local/bin/pwsh +pwsh --version 2>/dev/null || sudo chmod +x /usr/local/bin/pwsh LYQ=/usr/local/bin/yq sudo wget -qO $LYQ https://github.com/mikefarah/yq/releases/latest/download/yq_linux_arm64 sudo chmod +x $LYQ + +# Custom scripts +for fn in "$(find "${SHOVEL_HOME}/.devcontainer/" -iname '*.custom.*sh')"; do + chmod +x "$fn" + . "$fn" +done diff --git a/.gitignore b/.gitignore index d9995ac1c3..7f6c8621fe 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ checkver*.html *.iso /.cache/* /.vagrant/* -/.vscode/*.internal.code-workspace +/.vscode/*.custom.code-workspace +/.devcontainer/*.custom.*sh From b6f01a46102f6c86ae64e52fdda5ddce7b7447a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8C=C3=A1bera?= Date: Sun, 19 Feb 2023 18:57:56 +0000 Subject: [PATCH 10/10] devcontainer tweaks --- .devcontainer.json | 38 +++++++++++++++++++++----------------- lib/install.ps1 | 2 +- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.devcontainer.json b/.devcontainer.json index 396c9ba5de..0c89314809 100644 --- a/.devcontainer.json +++ b/.devcontainer.json @@ -11,23 +11,27 @@ "profile": "default" } }, - "extensions": [ - "DavidAnson.vscode-markdownlint", - "eamodio.gitlens", - "EditorConfig.EditorConfig", - "gruntfuggly.todo-tree", - "k--kato.intellij-idea-keybindings", - "medo64.render-crlf", - "ms-vscode.powershell-preview", - "redhat.vscode-yaml", - "usernamehw.errorlens", - "yzhang.markdown-all-in-one" - ], - "settings": { - "terminal.integrated.defaultProfile.linux": "zsh", - "powershell.powerShellDefaultVersion": "PowerShell Core (Linux)", - "powershell.powerShellAdditionalExePaths": { - "PowerShell Core (Linux)": "/usr/local/bin/pwsh" + "customizations": { + "vscode": { + "extensions": [ + "DavidAnson.vscode-markdownlint", + "eamodio.gitlens", + "EditorConfig.EditorConfig", + "gruntfuggly.todo-tree", + "k--kato.intellij-idea-keybindings", + "medo64.render-crlf", + "ms-vscode.powershell-preview", + "redhat.vscode-yaml", + "usernamehw.errorlens", + "yzhang.markdown-all-in-one" + ], + "settings": { + "terminal.integrated.defaultProfile.linux": "zsh", + "powershell.powerShellDefaultVersion": "PowerShell Core (Linux)", + "powershell.powerShellAdditionalExePaths": { + "PowerShell Core (Linux)": "/usr/local/bin/pwsh" + } + } } } } diff --git a/lib/install.ps1 b/lib/install.ps1 index f8c017eeda..1c96840582 100644 --- a/lib/install.ps1 +++ b/lib/install.ps1 @@ -683,7 +683,7 @@ function cookie_header($cookies) { } function is_in_dir($dir, $check) { - $check -match "^$([System.Text.RegularExpressions.Regex]::Escape("$dir"))(\\|`$)" + return $check -match "^$([System.Text.RegularExpressions.Regex]::Escape("$dir"))([\\/]|`$)" } function ftp_file_size($url) {