Skip to content

Commit 80cccfe

Browse files
JustinGroteJustinGrote
committed
Checkpoint for new dependency resolution method using pages
1 parent d54b07a commit 80cccfe

2 files changed

Lines changed: 185 additions & 50 deletions

File tree

‎ModuleFast.ps1‎

Lines changed: 167 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,12 @@ function Get-ModuleFastPlan {
7575
#This should enable HTTP/3 on Win11 22H2+ (or linux with http3 library) and PS 7.2+
7676
[void][AppContext]::SetSwitch('System.Net.SocketsHttpHandler.Http3Support', $true)
7777
}
78+
79+
#We pass this splat to all our HTTP requests to cut down on boilerplate
80+
$httpContext = @{
81+
HttpClient = $httpClient
82+
CancellationToken = $cancelToken.Token
83+
}
7884
# Write-Progress -Id 1 -Activity 'Get-ModuleFast' -CurrentOperation 'Fetching module information from Powershell Gallery'
7985
}
8086
PROCESS {
@@ -85,6 +91,7 @@ function Get-ModuleFastPlan {
8591
}
8692
}
8793
END {
94+
# A deduplicated list of modules to install
8895
[HashSet[ComparableModuleSpecification]]$modulesToInstall = @{}
8996

9097
# We use this as a fast lookup table for the context of the request
@@ -103,7 +110,7 @@ function Get-ModuleFastPlan {
103110
#TODO: Capture this somewhere that we can use it to report in the deploy plan
104111
continue
105112
}
106-
$task = Get-ModuleInfoAsync -Name $moduleSpec -HttpClient $httpclient -Uri $Source -CancellationToken $cancelToken.Token
113+
$task = Get-ModuleInfoAsync @httpContext -Endpoint $Source -Name $moduleSpec.Name
107114
$resolveTasks[$task] = $moduleSpec
108115
$currentTasks.Add($task)
109116
}
@@ -119,53 +126,112 @@ function Get-ModuleFastPlan {
119126
#TODO: Perform a HEAD query to see if something has changed
120127

121128
[Task[string]]$completedTask = $currentTasks[$thisTaskIndex]
122-
[ComparableModuleSpecification]$moduleSpec = $resolveTasks[$completedTask]
129+
[ComparableModuleSpecification]$currentModuleSpec = $resolveTasks[$completedTask]
123130

124-
Write-Debug "$moduleSpec`: Processing Response"
131+
Write-Debug "$currentModuleSpec`: Processing Response"
125132
# We use GetAwaiter so we get proper error messages back, as things such as network errors might occur here.
126133
#TODO: TryCatch logic for GetResult
127134
try {
128135
$response = $completedTask.GetAwaiter().GetResult()
129136
| ConvertFrom-Json
137+
Write-Debug "$currentModuleSpec`: Received Response with $($response.Count) pages"
130138
} catch {
131139
$taskException = $PSItem.Exception.InnerException
132140
#TODO: Rewrite this as a handle filter
133141
if ($taskException -isnot [HttpRequestException]) { throw }
134142
[HttpRequestException]$err = $taskException
135143
if ($err.StatusCode -eq [HttpStatusCode]::NotFound) {
136-
throw [InvalidOperationException]"$moduleSpec`: module was not found in the $Source repository. Check the spelling and try again."
144+
throw [InvalidOperationException]"$currentModuleSpec`: module was not found in the $Source repository. Check the spelling and try again."
137145
}
138146

139147
#All other cases
140-
$PSItem.ErrorDetails = "$moduleSpec`: Failed to fetch module $moduleSpec from $Source. Error: $PSItem"
148+
$PSItem.ErrorDetails = "$currentModuleSpec`: Failed to fetch module $currentModuleSpec from $Source. Error: $PSItem"
141149
throw $PSItem
142150
}
143151

144-
# HACK: Need to add @type to make this more discriminate between a direct version query and an individual item
145-
$responseItems = $response.catalogEntry ? $response.catalogEntry : $response.items.items.catalogEntry
152+
if (-not $response.count) {
153+
throw [InvalidDataException]"$currentModuleSpec`: invalid result received from $Source. This is probably a bug. Content: $response"
154+
}
146155

147-
#TODO: This should be a separate function with a pipeline
156+
#If what we are looking for exists in the response, we can stop looking
157+
#HACK: We are making a big assumption that the v3 API will always return the latest version(s) in the index
158+
#TODO: Check every version range that our selected item might not have a higher candidate in a non-inlined index
159+
#TODO: Type the responses and check on the type, not the existence of a property.
160+
$entries = $response.items.items.catalogEntry
161+
[version[]]$inlinedVersions = $entries.version
162+
| Where-Object {
163+
$PSItem -and !$PSItem.contains('-')
164+
}
165+
166+
[Version]$versionMatch = $moduleSpec.FindHighestMatch($inlinedVersions)
167+
if ($versionMatch) {
168+
Write-Debug "$currentModuleSpec`: Found satisfying version $versionMatch in the inlined index. TODO: Resolve dependencies"
169+
$selectedEntry = $entries | Where-Object version -EQ $versionMatch
170+
#TODO: Resolve dependencies in separate function
171+
} else {
172+
#Do a more detailed resolution
173+
Write-Debug "$currentModuleSpec`: not found in inlined index. Determining appropriate page(s) to query"
174+
throw [NotImplementedException]'TODO: Find and fetch additional info from the paging'
175+
#If not inlined, we need to find what page(s) might have the candidate info we are looking for.
176+
#While this may seem inefficient, all pages but latest are static and have a long lifetime so we trade a
177+
#longer cold start to a nearly infinite requery, which will handle all subsequent dependency lookups.
178+
#stats show most modules have a few common dependencies, so caching all versions of those dependencies is
179+
#very helpful for fast performance
180+
181+
# HACK: Need to add @type to make this more discriminate between a direct version query and an individual item
182+
# TODO: Should probably typesafe and validate this using classes
183+
184+
#Lookup that ties ranges to their page leaf ID
185+
[Dictionary[ComparableModuleSpecification, String]]$ranges = @{}
186+
foreach ($range in $response.items) {
187+
$spec = @{
188+
Name = $currentModuleSpec.Name
189+
}
190+
if ($range.lower -eq $range.upper) {
191+
$spec.RequiredVersion = $range.lower
192+
} else {
193+
$spec.MinimumVersion = $range.lower
194+
$spec.MaximumVersion = $range.upper
195+
}
196+
$ranges[$spec] = $range.'@id'
197+
}
198+
$pages = $moduleSpec.FindMatches($ranges.Keys).foreach{
199+
$ranges[$PSItem]
200+
}
201+
if (-not $pages) {
202+
throw [InvalidOperationException]"$currentModuleSpec`: a matching module was not found in the $Source repository that satisfies the version constraints. If this happens during dependency lookup, it is a bug in ModuleFast."
203+
}
204+
Write-Debug "$currentModuleSpec`: Found $($pages.Count) pages that might match the query."
205+
206+
#TODO: This is relatively slow and blocking, but we would need complicated logic to process it in the main task handler loop.
207+
#I really should make a pipeline that breaks off tasks based on the type of the response.
208+
#Good news is these pages should basically cache forever
209+
foreach ($page in $pages) {
210+
$task = Get-ModuleInfoAsync @httpContext -Endpoint $Source -Uri $page
211+
$resolveTasks[$task] = $currentModuleSpec
212+
$currentTasks.Add($task)
213+
}
214+
# [Version[]]$candidateVersions = $responseItems.Version
215+
# | Where-Object { -not $PSItem.contains('-') } #TODO: Support Prerelease
148216

149-
[Version[]]$candidateVersions = $responseItems.Version
150-
| Where-Object { -not $PSItem.contains('-') } #TODO: Support Prerelease
217+
# $selectedVersion = Find-HighestSatisfiesVersion $moduleSpec $candidateVersions
218+
# if (-not $selectedVersion) {
219+
# throw "No module that satisfies $moduleSpec was found in $Source"
220+
# }
151221

152-
$selectedVersion = Find-HighestSatisfiesVersion $moduleSpec $candidateVersions
153-
if (-not $selectedVersion) {
154-
throw "No module that satisfies $moduleSpec was found in $Source"
222+
# $selectedEntry = $responseItems | Where-Object Version -EQ $selectedVersion
155223
}
156224

157-
$selectedModule = $responseItems | Where-Object Version -EQ $selectedVersion
158-
if ($selectedModule.count -ne 1) {
159-
throw 'More than one selectedModule was specified. '
225+
if ($selectedEntry.count -ne 1) {
226+
throw 'Something other than exactly 1 selectedModule was specified. This should never happen and is a bug'
160227
}
161228

162229
$moduleInfo = [ComparableModuleSpecification]@{
163-
ModuleName = $selectedModule.id
164-
RequiredVersion = $selectedModule.version
230+
ModuleName = $selectedEntry.id
231+
RequiredVersion = $selectedEntry.version
165232
#TODO: Fix in Server API GUID = $moduleInfo.Guid
166233
}
167234

168-
169235
#Check if we have already processed this item and move on if we have
170236
if (-not $modulesToInstall.Add($moduleInfo)) {
171237
Write-Debug "$moduleInfo ModulesToInstall already exists. Skipping..."
@@ -175,22 +241,22 @@ function Get-ModuleFastPlan {
175241
continue
176242
}
177243

178-
Write-Verbose "$moduleInfo Added to ModulesToInstall."
244+
Write-Debug "$moduleInfo Added to ModulesToInstall."
179245

180246
# HACK: Pwsh doesn't care about target framework as of today so we can skip that evaluation
181247
# TODO: Should it? Should we check for the target framework and only install if it matches?
182-
$dependencyInfo = $selectedModule.dependencyGroups.dependencies
248+
$dependencyInfo = $selectedEntry.dependencyGroups.dependencies
183249

184250
#Determine dependencies and add them to the pending tasks
185251
if ($dependencyInfo) {
186252
# HACK: I should be using the Id provided by the server, for now I'm just guessing because
187253
# I need to add it to the ComparableModuleSpec class
188-
Write-Debug "$moduleSpec`: Processing dependencies"
254+
Write-Debug "$currentModuleSpec`: Processing dependencies"
189255
try {
190256
[List[ComparableModuleSpecification]]$dependencies = $dependencyInfo | Parse-NugetDependency
191257

192258
} catch { Wait-Debugger }
193-
Write-Debug "$moduleSpec has $($dependencies.count) dependencies"
259+
Write-Debug "$currentModuleSpec has $($dependencies.count) dependencies"
194260

195261
# TODO: Where loop filter maybe
196262
[ComparableModuleSpecification[]]$dependenciesToResolve = $dependencies | Where-Object {
@@ -249,8 +315,8 @@ function Get-ModuleFastPlan {
249315
# TODO: Deduplicate in-flight queries (az.accounts is a good example)
250316
# Write-Debug "$moduleSpec`: Checking if $dependencySpec already has an in-flight request that satisfies the requirement"
251317

252-
Write-Debug "$moduleSpec`: Fetching dependency $dependencySpec"
253-
$task = Get-ModuleInfoAsync -Name $dependencySpec -HttpClient $httpclient -Uri $Source -CancellationToken $cancelToken.Token
318+
Write-Debug "$currentModuleSpec`: Fetching dependency $dependencySpec"
319+
$task = Get-ModuleInfoAsync @httpContext -Endpoint $Source -Name $dependencySpec.Name
254320
$resolveTasks[$task] = $dependencySpec
255321
$currentTasks.Add($task)
256322
}
@@ -277,6 +343,12 @@ function Get-ModuleFastPlan {
277343
}
278344
# #endregion Main
279345

346+
#region PlanHelpers
347+
#endregion PlanHelpers
348+
349+
350+
351+
280352
#region Classes
281353
<#
282354
A custom version of ModuleSpecification that is comparable on its values, and will deduplicate in a HashSet if all
@@ -295,6 +367,7 @@ class ComparableModuleSpecification : ModuleSpecification {
295367
MaximumVersion = $spec.MaximumVersion
296368
}
297369
}
370+
298371
[ModuleSpecification] ToModuleSpecification() {
299372
return [ModuleSpecification]@{
300373
Name = $this.Name
@@ -305,6 +378,51 @@ class ComparableModuleSpecification : ModuleSpecification {
305378
}
306379
}
307380

381+
# Return only module specifications that match this specification. Good for determining which ranges will satisfy this module
382+
[ComparableModuleSpecification[]] FindMatch([ComparableModuleSpecification[]]$candidates) {
383+
$matchSpecs = foreach ($candidate in $candidates) {
384+
if ($this.Name -and ($this.Name -ne $candidate.Name)) {
385+
Write-Error ('The names of the module specifications you are trying to compare do not match ({0} vs {1})' -f $this.Name, $candidate.Name)
386+
continue
387+
}
388+
if ($this.Guid -and ($this.Guid -ne $candidate.Guid)) {
389+
Write-Error ('The GUIDs of the module specifications you are trying to compare do not match ({0} vs {1})' -f $this.Name, $candidate.Name)
390+
continue
391+
}
392+
if ($this.RequiredVersion -and ($this.RequiredVersion -ne $candidate.Version)) {
393+
continue
394+
}
395+
if ($this.Version -and ($candidate.Version -gt $this.Version )) {
396+
continue
397+
}
398+
if ($this.MaximumVersion -and ($this.MaximumVersion -lt $candidate.Version)) {
399+
continue
400+
}
401+
$candidate
402+
}
403+
404+
return $matchSpecs
405+
}
406+
407+
[Version[]] FindMatch([Version[]]$versions) {
408+
$candidates = foreach ($version in $versions) {
409+
[ComparableModuleSpecification]@{
410+
ModuleName = $this.Name
411+
RequiredVersion = $version
412+
}
413+
}
414+
return $this.FindMatch($candidates).RequiredVersion
415+
}
416+
417+
# Return the highest version that satisfies this module specification
418+
[ComparableModuleSpecification] FindHighestMatch([ComparableModuleSpecification[]]$candidates) {
419+
return $this.findMatch($candidates) | Sort-Object Version -Descending | Select-Object -First 1
420+
}
421+
422+
[Version] FindHighestMatch([Version[]]$versions) {
423+
return $this.findMatch($versions) | Sort-Object -Descending | Select-Object -First 1
424+
}
425+
308426
# Concatenate the properties into a string to generate a hashcode
309427
[int] GetHashCode() {
310428
return ($this.ToString()).GetHashCode()
@@ -315,6 +433,7 @@ class ComparableModuleSpecification : ModuleSpecification {
315433
[string] ToString() {
316434
return ($this.Name, $this.Guid, $this.Version, $this.MaximumVersion, $this.RequiredVersion -join ':')
317435
}
436+
318437
}
319438
#endRegion Classes
320439

@@ -509,39 +628,42 @@ function Install-Modulefast {
509628
}
510629
}
511630

512-
filter Get-ModuleInfoAsync {
631+
function Get-ModuleInfoAsync {
513632
[CmdletBinding()]
514633
[OutputType([Task[String]])]
515634
param (
516-
[Parameter(Mandatory)][ComparableModuleSpecification]$Name,
517-
[Parameter(Mandatory)][string]$Uri,
635+
# The name of the module to search for
636+
[Parameter(Mandatory, ParameterSetName = 'endpoint')][string]$Name,
637+
# The URI of the nuget v3 repository base, e.g. https://pwsh.gallery/index.json
638+
[Parameter(Mandatory, ParameterSetName = 'endpoint')]$Endpoint,
639+
# The path we are calling for the registration
640+
[Parameter(ParameterSetName = 'endpoint')][string]$Path = 'index.json',
641+
642+
#The direct URI to the registration endpoint
643+
[Parameter(Mandatory, ParameterSetName = 'uri')][string]$Uri,
644+
518645
[Parameter(Mandatory)][HttpClient]$HttpClient,
519-
[CancellationToken]$CancellationToken
646+
[Parameter(Mandatory)][CancellationToken]$CancellationToken
520647
)
521-
end {
648+
649+
if (-not $Uri) {
522650
$moduleSpec = $Name
523-
$ModuleId = $ModuleSpec.Name
651+
$ModuleId = $Name
524652

653+
#TODO: Call index.json and get the correct service. For now we are shortcutting this (bad behavior)
525654
#Strip any *.json path which might be at the end of the uri
526-
$endpoint = $Uri -replace '/\w+\.json$'
527655

528656
#HACK: We are making a *big* assumption here about the structure of the nuget repository to save an API call
529657
#TODO: Error handling if we are wrong by checking the main index.json manifest
530658

531-
if ($Name.MaximumVersion) {
532-
throw [NotImplementedException]"$Name has a maximum version. This is not implemented yet."
533-
}
534-
$moduleInfoUriBase = "$endpoint/registration/$ModuleId"
535-
if ($moduleSpec.RequiredVersion) {
536-
$moduleInfoUri = "$moduleInfoUriBase/$($moduleSpec.RequiredVersion).json"
537-
} else {
538-
$moduleInfoUri = "$moduleInfoUriBase/index.json"
539-
}
540-
541-
#TODO: System.Text.JSON serialize this with fancy generic methods in 7.3?
542-
Write-Debug "$ModuleId`: fetch info from $moduleInfoUri"
543-
return $HttpClient.GetStringAsync($moduleInfoUri, $CancellationToken)
659+
$endpointBase = $endpoint -replace '/\w+\.json$'
660+
$moduleInfoUriBase = "$endpointBase/registration/$ModuleId"
661+
$uri = "$moduleInfoUriBase/$Path"
544662
}
663+
664+
#TODO: System.Text.JSON serialize this with fancy generic methods in 7.3?
665+
Write-Debug ('{0}fetch info from {1}' -f ($ModuleId ? "$ModuleId`: " : ''), $uri)
666+
return $HttpClient.GetStringAsync($uri, $CancellationToken)
545667
}
546668

547669
filter Parse-NugetDependency ([Parameter(Mandatory, ValueFromPipeline)]$Dependency) {

‎ModuleFast.tests.ps1‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,28 @@
11
BeforeAll {
22
. ./ModuleFast.ps1
3-
}
4-
Describe 'Get-ModuleFastPlan' {
5-
It 'Gets Module' {
6-
Get-ModuleFastPlan 'ImportExcel' | Should -HaveCount 1
3+
if ($env:MFURI) {
4+
$debugPreference = 'continue'
5+
$verbosePreference = 'continue'
6+
$PSDefaultParameterValues['Get-ModuleFastPlan:Source'] = $env:MFURI
77
}
8+
}
9+
Describe 'Get-ModuleFastPlan' -Tag 'E2E' {
10+
$SCRIPT:moduleName = 'Az.Accounts'
11+
It 'Gets Module by <Test>' {
12+
$actual = Get-ModuleFastPlan $spec
13+
$actual | Should -HaveCount 1
14+
$actual.Name | Should -Be $moduleName
15+
$actual.RequiredVersion -as [Version] | Should -Not -BeNullOrEmpty
16+
} -TestCases (
17+
@{Test = 'Name'; Spec = $moduleName },
18+
@{Test = 'MinimumVersion'; Spec = @{ ModuleName = $moduleName; ModuleVersion = '0.0.0' } },
19+
@{Test = 'RequiredVersionNotLatest'; Spec = @{ ModuleName = $moduleName; RequiredVersion = '2.7.3' } }
20+
)
821
It 'Gets Module with 1 dependency' {
922
Get-ModuleFastPlan 'Az.Compute' | Should -HaveCount 2
1023
}
1124
It 'Gets Module with lots of dependencies (Az)' {
1225
#TODO: Mocks
13-
Get-ModuleFastPlan 'Az' | Should -HaveCount 77
26+
Get-ModuleFastPlan 'Az' | Should -HaveCount 78
1427
}
1528
}

0 commit comments

Comments
 (0)