Skip to content

Commit 950b5be

Browse files
JustinGroteJustinGrote
committed
Change Extraction Method
1 parent 766b10e commit 950b5be

2 files changed

Lines changed: 61 additions & 57 deletions

File tree

‎ModuleFast.ps1‎

Lines changed: 58 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ using namespace System.Collections.Concurrent
44
using namespace System.Collections.Generic
55
using namespace System.Collections.Specialized
66
using namespace System.IO
7+
using namespace System.IO.Compression
78
using namespace System.IO.Pipelines
89
using namespace System.Management.Automation
910
using namespace System.Net
@@ -19,8 +20,6 @@ High Performance Powershell Module Installation
1920
THIS IS NOT FOR PRODUCTION, it should be considered "Fragile" and has very little error handling and type safety
2021
It also doesn't generate the PowershellGet XML files currently, so PowershellGet will see them as "External" modules
2122
#>
22-
23-
2423
function Install-ModuleFast {
2524
[CmdletBinding(SupportsShouldProcess)]
2625
param(
@@ -459,9 +458,6 @@ function Install-ModuleFastHelper {
459458
[Dictionary[Task, hashtable]]$taskMap = @{}
460459

461460
[List[Task[Stream]]]$streamTasks = foreach ($module in $ModuleToInstall) {
462-
#TODO: GetStreamAsync then waitone on those and connect them to a file stream, then threadjob the install
463-
#since zip is sync blocking
464-
#TODO: Check file health and integrity. If it's good, skip the download and move on to install process.
465461
$context = @{
466462
Module = $module
467463
DownloadPath = Join-Path $ModuleCache "$($module.Name).$($module.Version).nupkg"
@@ -470,7 +466,7 @@ function Install-ModuleFastHelper {
470466
$taskMap.Add($fetchTask, $context)
471467
$fetchTask
472468
}
473-
469+
[List[Job2]]$installJobs = @()
474470
[List[Task]]$downloadTasks = while ($streamTasks.count -gt 0) {
475471
$noTasksYetCompleted = -1
476472
[int]$thisTaskIndex = [Task]::WaitAny($streamTasks, 500)
@@ -480,58 +476,21 @@ function Install-ModuleFastHelper {
480476
$context = $taskMap[$thisTask]
481477
$context.fetchStream = $stream
482478
$streamTasks.RemoveAt($thisTaskIndex)
483-
$downloadPath = $context.DownloadPath
484-
[void][directory]::CreateDirectory((Split-Path $downloadPath))
485-
$destFileStream = [file]::Create($DownloadPath, 4096, [FileOptions]::SequentialScan)
486-
$context.fileStream = $destFileStream
487-
$downloadedTask = $stream.CopyToAsync($destFileStream, $CancellationToken)
488-
$taskMap.Add($downloadedTask, $context)
489-
$downloadedTask
490-
}
491479

492-
#Installation jobs are captured here, we will check them once all downloads have completed
493-
[List[Job2]]$installJobs = @()
480+
#We are going to extract these straight out of memory, so we don't need to write the nupkg to disk
494481

495-
$downloaded = 0
496-
$downloadedProgressId = Get-Random
497-
#TODO: Filestreams should be disposed in a try/catch in case of cancellation. In PS 7.3+, should be a clean() block
498-
while ($downloadTasks.count -gt 0) {
499-
#TODO: Check on in jobs and if there's a failure, cancel the rest of the jobs
500-
$noTasksYetCompleted = -1
501-
[int]$thisTaskIndex = [Task]::WaitAny($downloadTasks, 500)
502-
if ($thisTaskIndex -eq $noTasksYetCompleted) { continue }
503-
$thisTask = $downloadTasks[$thisTaskIndex]
504-
$context = $taskMap[$thisTask]
505-
# We can close these streams now that it is downloaded.
506-
# This releases the lock on the file
507-
#TODO: Maybe can connect the stream to a zip decompressionstream. Should be in cache so performance would be negligible
508-
$context.fileStream.Dispose()
509-
$context.fetchStream.Dispose()
510-
#The file copy task is a void task that doesnt return anything, so we dont need to do GetResult()
511-
$downloadTasks.RemoveAt($thisTaskIndex)
512-
513-
#Start a new threadjob to handle the installation, because the zipfile API is not async. Also extraction is
514-
#CPU intensive so multiple threads will be helpful here and worth the startup cost of a runspace
515-
$installJobParams = @{
516-
ScriptBlock = (Get-Item Function:\Install-ModuleFastOperation).Scriptblock
517-
#Named parameters require a hack so we will just do these in order
518-
ArgumentList = @(
519-
$context.Module.Name,
520-
$context.Module.Version,
521-
$context.DownloadPath,
522-
$Destination
523-
)
482+
$installPath = Join-Path $Destination $context.Module.Name $context.Module.Version
483+
Write-Verbose "Starting Extract Job $($context.Module) to $installPath"
484+
# This is a sync process and we want to do it in parallel, hence the threadjob
485+
$installJob = Start-ThreadJob -ThrottleLimit 8 {
486+
$zip = [IO.Compression.ZipArchive]::new($USING:stream, 'Read')
487+
[IO.Compression.ZipFileExtensions]::ExtractToDirectory($zip, $USING:installPath)
488+
($zip).Dispose()
489+
($USING:stream).Dispose()
524490
}
525-
Write-Debug "Starting Module Install Job for $($context.Module)"
526-
$installJob = Start-ThreadJob @installJobParams
527491
$installJobs.Add($installJob)
528-
$downloaded++
529-
Write-Progress -Id $downloadedProgressId -ParentId 1 -Activity 'Download' -Status "$downloaded/$($ModuleToInstall.count) Modules" -PercentComplete ($downloaded / $ModuleToInstall.count * 100)
530-
531492
}
532493

533-
#TODO: Correlate the installjobs to a dictionary so we can return the original modulespec maybe?
534-
#Or is that even needed?
535494
$installed = 0
536495
$installProgressId = (Get-Random)
537496
while ($installJobs.count -gt 0) {
@@ -542,6 +501,51 @@ function Install-ModuleFastHelper {
542501
$installed++
543502
Write-Progress -Id $installProgressId -ParentId 1 -Activity 'Install' -Status "$installed/$($ModuleToInstall.count) Modules" -PercentComplete ($installed / $ModuleToInstall.count * 100)
544503
}
504+
505+
# #Installation jobs are captured here, we will check them once all downloads have completed
506+
507+
508+
# $downloaded = 0
509+
# $downloadedProgressId = Get-Random
510+
# #TODO: Filestreams should be disposed in a try/catch in case of cancellation. In PS 7.3+, should be a clean() block
511+
# while ($downloadTasks.count -gt 0) {
512+
# #TODO: Check on in jobs and if there's a failure, cancel the rest of the jobs
513+
# $noTasksYetCompleted = -1
514+
# [int]$thisTaskIndex = [Task]::WaitAny($downloadTasks, 500)
515+
# if ($thisTaskIndex -eq $noTasksYetCompleted) { continue }
516+
# $thisTask = $downloadTasks[$thisTaskIndex]
517+
# $context = $taskMap[$thisTask]
518+
# # We can close these streams now that it is downloaded.
519+
# # This releases the lock on the file
520+
# #TODO: Maybe can connect the stream to a zip decompressionstream. Should be in cache so performance would be negligible
521+
# $context.fileStream.Dispose()
522+
# $context.fetchStream.Dispose()
523+
# #The file copy task is a void task that doesnt return anything, so we dont need to do GetResult()
524+
# $downloadTasks.RemoveAt($thisTaskIndex)
525+
526+
# #Start a new threadjob to handle the installation, because the zipfile API is not async. Also extraction is
527+
# #CPU intensive so multiple threads will be helpful here and worth the startup cost of a runspace
528+
# $installJobParams = @{
529+
# ScriptBlock = (Get-Item Function:\Install-ModuleFastOperation).Scriptblock
530+
# #Named parameters require a hack so we will just do these in order
531+
# ArgumentList = @(
532+
# $context.Module.Name,
533+
# $context.Module.Version,
534+
# $context.DownloadPath,
535+
# $Destination
536+
# )
537+
# }
538+
# Write-Debug "Starting Module Install Job for $($context.Module)"
539+
# $installJob = Start-ThreadJob @installJobParams
540+
# $installJobs.Add($installJob)
541+
# $downloaded++
542+
# Write-Progress -Id $downloadedProgressId -ParentId 1 -Activity 'Download' -Status "$downloaded/$($ModuleToInstall.count) Modules" -PercentComplete ($downloaded / $ModuleToInstall.count * 100)
543+
544+
# }
545+
546+
# #TODO: Correlate the installjobs to a dictionary so we can return the original modulespec maybe?
547+
# #Or is that even needed?
548+
545549
}
546550

547551
# This will be run inside a threadjob. We separate this so that we can test it independently
@@ -1103,7 +1107,7 @@ function Find-LocalModule {
11031107
# BUG: Prerelease Module paths are still not recognized by internal PS commands and can break things
11041108

11051109
# Search all psmodulepaths for the module
1106-
$modulePaths = $env:PSModulePath -split ([Path]::PathSeparator, [StringSplitOptions]::RemoveEmptyEntries)
1110+
$modulePaths = $env:PSModulePath.Split([Path]::PathSeparator, [StringSplitOptions]::RemoveEmptyEntries)
11071111
if (-Not $modulePaths) {
11081112
Write-Warning 'No PSModulePaths found in $env:PSModulePath. If you are doing isolated testing you can disregard this.'
11091113
return
@@ -1221,6 +1225,7 @@ function ConvertTo-AuthenticationHeaderValue ([PSCredential]$Credential) {
12211225
)
12221226
return [Net.Http.Headers.AuthenticationHeaderValue]::New('Basic', $basicCredential)
12231227
}
1228+
12241229
#endregion Helpers
12251230

12261231
# Export-ModuleMember Get-ModuleFast

‎ModuleFast.tests.ps1‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -362,18 +362,17 @@ Describe 'Install-ModuleFast' -Tag 'E2E' {
362362
$SCRIPT:__existingPSModulePath = $env:PSModulePath
363363
}
364364
It 'Installs Module' {
365-
$env:PSModulePath = ''
366365
#HACK: The testdrive mount is not available in the threadjob runspaces so we need to translate it
367366
$testDrivePath = (Get-Item testdrive:).fullname
368-
Install-ModuleFast 'Az.Accounts' -Destination $testDrivePath
367+
Install-ModuleFast 'Az.Accounts' -Destination $testDrivePath -NoProfileUpdate -NoPSModulePathUpdate
369368
Get-Item TestDrive:\Az.Accounts\*\Az.Accounts.psd1 | Should -Not -BeNullOrEmpty
370369
}
371370
It 'Installs Module with lots of dependencies (Az)' {
372-
Install-ModuleFast 'Az' -Destination (Get-Item testdrive:).fullname
371+
Install-ModuleFast 'Az' -Destination (Get-Item testdrive:).fullname -NoProfileUpdate -NoPSModulePathUpdate
373372
}
374373
It 'Installs Module with 4 section version numbers (VMware.PowerCLI)' {
375374
$testDrivePath = (Get-Item testdrive:).fullname
376-
Install-ModuleFast 'VMware.VimAutomation.Common' -Destination $testDrivePath
375+
Install-ModuleFast 'VMware.VimAutomation.Common' -Destination $testDrivePath -NoProfileUpdate -NoPSModulePathUpdate
377376
Get-Item TestDrive:\*\*\*.psd1 | ForEach-Object {
378377
$moduleFolderVersion = $_ | Split-Path | Split-Path -Leaf
379378
Import-PowerShellDataFile -Path $_.FullName | ForEach-Object ModuleVersion | Should -Be $moduleFolderVersion

0 commit comments

Comments
 (0)