Skip to content

Commit d54b07a

Browse files
JustinGroteJustinGrote
committed
Rework Module Query Logic
1 parent a55b2d0 commit d54b07a

2 files changed

Lines changed: 130 additions & 110 deletions

File tree

‎ModuleFast.ps1‎

Lines changed: 122 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ function Get-ModuleFastPlan {
3535
#Whether to include prerelease modules in the request
3636
[Switch]$PreRelease,
3737
#By default we use in-place modules if they satisfy the version requirements. This switch will force a search for all latest modules
38-
[Switch]$Latest
38+
[Switch]$Update
3939
)
4040

4141
BEGIN {
@@ -58,17 +58,21 @@ function Get-ModuleFastPlan {
5858
#This is not as big of a deal as it used to be.
5959
$SCRIPT:httpClient = [HttpClient]::new($httpHandler)
6060
$httpClient.BaseAddress = $Source
61+
62+
#This user agent is important, it indicates to pwsh.gallery that we want dependency-only metadata
63+
#TODO: Do this with a custom header instead
6164
$userHeaderAdded = $httpClient.DefaultRequestHeaders.UserAgent.TryParseAdd('ModuleFast (https://gist.github.com/JustinGrote/ecdf96b4179da43fb017dccbd1cc56f6)')
6265
if (-not $userHeaderAdded) {
6366
throw 'Failed to add User-Agent header to HttpClient. This is a bug'
6467
}
65-
#For now, drop support for HTTP/1.1 (curious if this will be a problem)
68+
#TODO: Add switch to force HTTP/2. Most of the time it should work fine tho
69+
# $httpClient.DefaultRequestVersion = '2.0'
70+
#I tried to drop support for HTTP/1.1 but proxies and cloudflare testing still require it
71+
6672
#This will multiplex all queries over a single connection, minimizing TLS setup overhead
6773
#Should also support HTTP/3 on newest PS versions
68-
#Disabled for now as CF worker troubleshooting seems to have an issue maybe
69-
# $httpClient.DefaultRequestVersion = '2.0'
7074
$httpClient.DefaultVersionPolicy = [HttpVersionPolicy]::RequestVersionOrHigher
71-
#This should enable HTTP/3 on Win11 22H2+ and PS 7.2+
75+
#This should enable HTTP/3 on Win11 22H2+ (or linux with http3 library) and PS 7.2+
7276
[void][AppContext]::SetSwitch('System.Net.SocketsHttpHandler.Http3Support', $true)
7377
}
7478
# Write-Progress -Id 1 -Activity 'Get-ModuleFast' -CurrentOperation 'Fetching module information from Powershell Gallery'
@@ -126,8 +130,8 @@ function Get-ModuleFastPlan {
126130
} catch {
127131
$taskException = $PSItem.Exception.InnerException
128132
#TODO: Rewrite this as a handle filter
129-
if ($PSItem.Exception.InnerException -isnot [HttpRequestException]) { throw }
130-
[HttpRequestException]$err = $PSItem.Exception.InnerException
133+
if ($taskException -isnot [HttpRequestException]) { throw }
134+
[HttpRequestException]$err = $taskException
131135
if ($err.StatusCode -eq [HttpStatusCode]::NotFound) {
132136
throw [InvalidOperationException]"$moduleSpec`: module was not found in the $Source repository. Check the spelling and try again."
133137
}
@@ -141,103 +145,114 @@ function Get-ModuleFastPlan {
141145
$responseItems = $response.catalogEntry ? $response.catalogEntry : $response.items.items.catalogEntry
142146

143147
#TODO: This should be a separate function with a pipeline
144-
foreach ($moduleInfo in $responseItems) {
145-
# FIXME: Support Preview Releases
146-
if ($moduleInfo.version.contains('-')) {
147-
Write-Verbose "WARN: Skipping $($moduleInfo.id) $($moduleInfo.version) candidate because it is a preview release. This will be implemented later"
148-
continue
149-
}
150148

151-
#Create a module spec with our returned info
152-
[ComparableModuleSpecification]$moduleSpec = @{
153-
ModuleName = $moduleInfo.id
154-
RequiredVersion = $moduleInfo.version
155-
#TODO: Fix in Server API GUID = $moduleInfo.Guid
149+
[Version[]]$candidateVersions = $responseItems.Version
150+
| Where-Object { -not $PSItem.contains('-') } #TODO: Support Prerelease
151+
152+
$selectedVersion = Find-HighestSatisfiesVersion $moduleSpec $candidateVersions
153+
if (-not $selectedVersion) {
154+
throw "No module that satisfies $moduleSpec was found in $Source"
155+
}
156+
157+
$selectedModule = $responseItems | Where-Object Version -EQ $selectedVersion
158+
if ($selectedModule.count -ne 1) {
159+
throw 'More than one selectedModule was specified. '
160+
}
161+
162+
$moduleInfo = [ComparableModuleSpecification]@{
163+
ModuleName = $selectedModule.id
164+
RequiredVersion = $selectedModule.version
165+
#TODO: Fix in Server API GUID = $moduleInfo.Guid
166+
}
167+
168+
169+
#Check if we have already processed this item and move on if we have
170+
if (-not $modulesToInstall.Add($moduleInfo)) {
171+
Write-Debug "$moduleInfo ModulesToInstall already exists. Skipping..."
172+
#TODO: Fix the flow so this isn't stated twice
173+
[void]$resolveTasks.Remove($completedTask)
174+
[void]$currentTasks.Remove($completedTask)
175+
continue
176+
}
177+
178+
Write-Verbose "$moduleInfo Added to ModulesToInstall."
179+
180+
# HACK: Pwsh doesn't care about target framework as of today so we can skip that evaluation
181+
# TODO: Should it? Should we check for the target framework and only install if it matches?
182+
$dependencyInfo = $selectedModule.dependencyGroups.dependencies
183+
184+
#Determine dependencies and add them to the pending tasks
185+
if ($dependencyInfo) {
186+
# HACK: I should be using the Id provided by the server, for now I'm just guessing because
187+
# I need to add it to the ComparableModuleSpec class
188+
Write-Debug "$moduleSpec`: Processing dependencies"
189+
try {
190+
[List[ComparableModuleSpecification]]$dependencies = $dependencyInfo | Parse-NugetDependency
191+
192+
} catch { Wait-Debugger }
193+
Write-Debug "$moduleSpec has $($dependencies.count) dependencies"
194+
195+
# TODO: Where loop filter maybe
196+
[ComparableModuleSpecification[]]$dependenciesToResolve = $dependencies | Where-Object {
197+
# TODO: This dependency resolution logic should be a separate function
198+
# Maybe ModulesToInstall should be nested/grouped by Module Name then version to speed this up, as it currently
199+
# enumerates every time which shouldn't be a big deal for small dependency trees but might be a
200+
# meaninful performance difference on a whole-system upgrade.
201+
[HashSet[string]]$moduleNames = $modulesToInstall.Name
202+
if ($PSItem.Name -notin $ModuleNames) {
203+
Write-Debug "$PSItem not already in ModulesToInstall. Resolving..."
204+
return $true
205+
}
206+
207+
$plannedVersions = $modulesToInstall
208+
| Where-Object Name -EQ $PSItem.Name
209+
| Sort-Object RequiredVersion -Descending
210+
211+
# TODO: Consolidate with Get-HighestSatisfiesVersion function
212+
$highestPlannedVersion = $plannedVersions[0].RequiredVersion
213+
214+
if ($PSItem.Version -and ($PSItem.Version -gt $highestPlannedVersion)) {
215+
Write-Debug "$($PSItem.Name): Minimum Version $($PSItem.Version) not satisfied by highest existing match $highestPlannedVersion. Performing Lookup."
216+
return $true
217+
}
218+
219+
if ($PSItem.MaximumVersion -and ($PSItem.MaximumVersion -lt $highestPlannedVersion)) {
220+
Write-Debug "$($PSItem.Name): $highestPlannedVersion is higher than Maximum Version $($PSItem.MaximumVersion). Performing Lookup"
221+
return $true
222+
}
223+
224+
if ($PSItem.RequiredVersion -and ($PSItem.RequiredVersion -notin $plannedVersions.RequiredVersion)) {
225+
Write-Debug "$($PSItem.Name): Explicity Required Version $($PSItem.RequiredVersion) is not within existing planned versions ($($plannedVersions.RequiredVersion -join ',')). Performing Lookup"
226+
return $true
227+
}
228+
229+
#If it didn't match, skip it
230+
Write-Debug "$($PSItem.Name) dependency satisfied by $highestPlannedVersion already in the plan"
156231
}
157232

158-
#Check if we have already processed this item and move on if we have
159-
if (-not $modulesToInstall.Add($moduleSpec)) {
160-
Write-Debug "$ModuleSpec ModulesToInstall already exists. Skipping..."
233+
if (-not $dependenciesToResolve) {
234+
Write-Debug "$moduleSpec has no remaining dependencies that need resolving"
161235
continue
162236
}
163237

164-
Write-Debug "$moduleSpec Added to ModulesToInstall."
165-
166-
# HACK: Pwsh doesn't care about target framework as of today so we can skip that evaluation
167-
# TODO: Should it? Should we check for the target framework and only install if it matches?
168-
$dependencyInfo = $moduleInfo.dependencyGroups.dependencies
169-
170-
#Determine dependencies and add them to the pending tasks
171-
if ($dependencyInfo) {
172-
# HACK: I should be using the Id provided by the server, for now I'm just guessing because
173-
# I need to add it to the ComparableModuleSpec class
174-
Write-Debug "$moduleSpec`: Processing dependencies"
175-
try {
176-
[List[ComparableModuleSpecification]]$dependencies = $dependencyInfo | Parse-NugetDependency
177-
178-
} catch { Wait-Debugger }
179-
Write-Debug "$moduleSpec has $($dependencies.count) dependencies"
180-
181-
# TODO: Where loop filter maybe
182-
[ComparableModuleSpecification[]]$dependenciesToResolve = $dependencies | Where-Object {
183-
# TODO: This dependency resolution logic should be a separate function
184-
# Maybe ModulesToInstall should be nested/grouped by Module Name then version to speed this up, as it currently
185-
# enumerates every time which shouldn't be a big deal for small dependency trees but might be a
186-
# meaninful performance difference on a whole-system upgrade.
187-
[HashSet[string]]$moduleNames = $modulesToInstall.Name
188-
if ($PSItem.Name -notin $ModuleNames) {
189-
Write-Debug "$PSItem not already in ModulesToInstall. Resolving..."
190-
return $true
191-
}
192-
193-
$plannedVersions = $modulesToInstall
194-
| Where-Object Name -EQ $PSItem.Name
195-
| Sort-Object RequiredVersion -Descending
196-
197-
# TODO: Consolidate with Get-HighestSatisfiesVersion function
198-
$highestPlannedVersion = $plannedVersions[0].RequiredVersion
199-
200-
if ($PSItem.Version -and ($PSItem.Version -gt $highestPlannedVersion)) {
201-
Write-Debug "$($PSItem.Name): Minimum Version $($PSItem.Version) not satisfied by highest existing match $highestPlannedVersion. Performing Lookup."
202-
return $true
203-
}
204-
205-
if ($PSItem.MaximumVersion -and ($PSItem.MaximumVersion -lt $highestPlannedVersion)) {
206-
Write-Debug "$($PSItem.Name): $highestPlannedVersion is higher than Maximum Version $($PSItem.MaximumVersion). Performing Lookup"
207-
return $true
208-
}
209-
210-
if ($PSItem.RequiredVersion -and ($PSItem.RequiredVersion -notin $plannedVersions.RequiredVersion)) {
211-
Write-Debug "$($PSItem.Name): Explicity Required Version $($PSItem.RequiredVersion) is not within existing planned versions ($($plannedVersions.RequiredVersion -join ',')). Performing Lookup"
212-
return $true
213-
}
214-
215-
#If it didn't match, skip it
216-
Write-Debug "$($PSItem.Name) dependency satisfied by $highestPlannedVersion already in the plan"
217-
}
238+
Write-Debug "Fetching info on remaining $($dependenciesToResolve.count) dependencies"
218239

219-
if (-not $dependenciesToResolve) {
220-
Write-Debug "$moduleSpec has no remaining dependencies that need resolving"
240+
# We do this here rather than populate modulesToResolve because the tasks wont start until all the existing tasks complete
241+
# TODO: Figure out a way to dedupe this logic maybe recursively but I guess a function would be fine too
242+
foreach ($dependencySpec in $dependenciesToResolve) {
243+
$localMatch = Find-LocalModule $dependencySpec
244+
if ($localMatch) {
245+
Write-Verbose "Found local module $localMatch that satisfies dependency $dependencySpec. Skipping..."
246+
#TODO: Capture this somewhere that we can use it to report in the deploy plan
221247
continue
222248
}
249+
# TODO: Deduplicate in-flight queries (az.accounts is a good example)
250+
# Write-Debug "$moduleSpec`: Checking if $dependencySpec already has an in-flight request that satisfies the requirement"
223251

224-
Write-Debug "Fetching info on remaining $($dependenciesToResolve.count) dependencies"
225-
226-
# We do this here rather than populate modulesToResolve because the tasks wont start until all the existing tasks complete
227-
# TODO: Figure out a way to dedupe this logic maybe recursively but I guess a function would be fine too
228-
foreach ($dependencySpec in $dependenciesToResolve) {
229-
$localMatch = Find-LocalModule $dependencySpec
230-
if ($localMatch) {
231-
Write-Verbose "Found local module $localMatch that satisfies dependency $dependencySpec. Skipping..."
232-
#TODO: Capture this somewhere that we can use it to report in the deploy plan
233-
continue
234-
}
235-
Write-Debug "$moduleSpec`: Fetching dependency $dependencySpec"
236-
$task = Get-ModuleInfoAsync -Name $dependencySpec -HttpClient $httpclient -Uri $Source -CancellationToken $cancelToken.Token
237-
$resolveTasks[$task] = $dependencySpec
238-
$currentTasks.Add($task)
239-
}
240-
252+
Write-Debug "$moduleSpec`: Fetching dependency $dependencySpec"
253+
$task = Get-ModuleInfoAsync -Name $dependencySpec -HttpClient $httpclient -Uri $Source -CancellationToken $cancelToken.Token
254+
$resolveTasks[$task] = $dependencySpec
255+
$currentTasks.Add($task)
241256
}
242257
}
243258
try {
@@ -494,15 +509,6 @@ function Install-Modulefast {
494509
}
495510
}
496511

497-
function Get-NotInstalledModules ([String[]]$Name) {
498-
$InstalledModules = Get-Module $Name -ListAvailable
499-
$Name.where{
500-
$isInstalled = $PSItem -notin $InstalledModules.Name
501-
if ($isInstalled) { Write-Verbose "$PSItem is already installed. Skipping..." }
502-
return $isInstalled
503-
}
504-
}
505-
506512
filter Get-ModuleInfoAsync {
507513
[CmdletBinding()]
508514
[OutputType([Task[String]])]
@@ -666,7 +672,7 @@ function Find-LocalModule {
666672
Write-Verbose "$moduleSpec`: module folder exists at $moduleNamePath but no modules found that match the version spec."
667673
continue
668674
}
669-
$versionMatch = Get-HighestSatisfiesVersion -ModuleSpec $ModuleSpec -Version $candidateVersions
675+
$versionMatch = Find-HighestSatisfiesVersion -ModuleSpec $ModuleSpec -Version $candidateVersions
670676
if ($versionMatch) {
671677
$manifestPath = Join-Path $moduleNamePath $([Version]$versionMatch) "$($ModuleSpec.Name).psd1"
672678
if (-not [File]::Exists($manifestPath)) {
@@ -685,11 +691,11 @@ function Find-LocalModule {
685691
<#
686692
Given an array of versions, find the highest one that satisfies the module spec. Returns $false if no match is found.
687693
#>
688-
function Get-HighestSatisfiesVersion {
694+
function Find-HighestSatisfiesVersion {
689695
param(
690696
[Parameter(Mandatory)][ComparableModuleSpecification]$ModuleSpec,
691697
#Versions that are potential candidates to satisfy the modulespec
692-
[Parameter(Mandatory)][HashSet[Version]]$Version
698+
[Parameter(Mandatory)][HashSet[Version]]$Versions
693699
)
694700
# TODO: Semver-compatible version of this function
695701

@@ -719,6 +725,16 @@ function Get-HighestSatisfiesVersion {
719725
}
720726
}
721727

728+
#BUG: This is required because the SMA.SemanticVersion class cannot handle build (+) by itself
729+
#https://github.com/PowerShell/PowerShell/issues/14605
730+
function ConvertTo-Version([SemanticVersion]$Version, [string]$BuildHint = 'SEMBUILD') {
731+
if ($null -eq ($Version.BuildLabel -as [int])) {
732+
Write-Warning [InvalidDataException]"BuildLabel $($Version.BuildLabel) is not numeric and cannot be cast, and will be skipped."
733+
[Version]::new($Version.Major, $Version.Minor, $Version.Patch)
734+
}
735+
[Version]::new($Version.Major, $Version.Minor, $Version.Patch)
736+
}
737+
722738
#endregion Helpers
723739

724740
# Export-ModuleMember Get-ModuleFast

0 commit comments

Comments
 (0)