From c1009c3d7cf63371a7ca9ff5f89e8c4ec0469191 Mon Sep 17 00:00:00 2001 From: "Mr.k" Date: Tue, 10 Sep 2024 04:05:10 +0300 Subject: [PATCH] Improve preprocessor (#2579) * Update documentation for 'Invoke-Preprocessing' Script Tool * Improve Compile Script a bit Deduplicating a lot of un-needed pre-fixes - Improve implementation for 'Invoke-Preprocessing' Script Tool * Fix RegEx in 'Invoke-Preprocessing' Script Tool * Result of Preprocessing * Update Replace Regex for Code Formatting in 'Invoke-Preprocessing' Script Tool * Result of Preprocessing * Update Exclude Files List for Preprocessing in 'Compile.ps1' Script * Remove Extra Whitespace in some place for 'Invoke-Preprocessing.ps1' Script Tool * Simplified and Improved the Exclude List Validation Step in 'Invoke-Preprocessing.ps1' Script Tool * Restore 'workingdir' variable when using '-Run' Parameter with 'Compile.ps1' Script * Revert "Update Exclude Files List for Preprocessing in 'Compile.ps1' Script" This reverts commit 674ab0308b71fd6c3215922268c6979f50425a11. * Result of Preprocessing --- .github/workflows/pre-release.yaml | 2 +- Compile.ps1 | 35 ++++--- config/autounattend.xml | 2 +- functions/private/Invoke-WinUtilGPU.ps1 | 2 +- .../private/Invoke-WinUtilMicroWin-Helper.ps1 | 2 +- scripts/main.ps1 | 6 +- tools/Invoke-Preprocessing.ps1 | 97 +++++++++++++------ 7 files changed, 96 insertions(+), 50 deletions(-) diff --git a/.github/workflows/pre-release.yaml b/.github/workflows/pre-release.yaml index a064f8a386..39a89d14ea 100644 --- a/.github/workflows/pre-release.yaml +++ b/.github/workflows/pre-release.yaml @@ -91,7 +91,7 @@ jobs: name: Pre-Release ${{ env.version }} body: | ${{ steps.generate_notes.outputs.body }} - + ![GitHub Downloads (specific asset, specific tag)](https://img.shields.io/github/downloads/ChrisTitusTech/winutil/${{ env.VERSION }}/winutil.ps1) append_body: false files: ./winutil.ps1 diff --git a/Compile.ps1 b/Compile.ps1 index afbe943d49..c9d4355587 100644 --- a/Compile.ps1 +++ b/Compile.ps1 @@ -8,6 +8,9 @@ $OFS = "`r`n" $scriptname = "winutil.ps1" $workingdir = $PSScriptRoot +Push-Location +Set-Location $workingdir + # Variable to sync between runspaces $sync = [Hashtable]::Synchronized(@{}) $sync.PSScriptRoot = $workingdir @@ -42,11 +45,11 @@ if (-NOT $SkipPreprocessing) { # Dot source the 'Invoke-Preprocessing' Function from 'tools/Invoke-Preprocessing.ps1' Script $preprocessingFilePath = ".\tools\Invoke-Preprocessing.ps1" - . "$(($workingdir -replace ('\\$', '')) + '\' + ($preprocessingFilePath -replace ('\.\\', '')))" + . $preprocessingFilePath $excludedFiles = @('.\.git\', '.\.gitignore', '.\.gitattributes', '.\.github\CODEOWNERS', '.\LICENSE', "$preprocessingFilePath", '*.png', '*.exe') $msg = "Pre-req: Code Formatting" - Invoke-Preprocessing -WorkingDir "$workingdir" -ExcludedFiles $excludedFiles -ProgressStatusMessage $msg + Invoke-Preprocessing -WorkingDir "$workingdir" -ExcludedFiles $excludedFiles -ProgressStatusMessage $msg -ThrowExceptionOnEmptyFilesList } # Create the script in memory. @@ -57,14 +60,14 @@ Update-Progress "Adding: Header" 5 $script_content.Add($header) Update-Progress "Adding: Version" 10 -$script_content.Add($(Get-Content "$workingdir\scripts\start.ps1").replace('#{replaceme}',"$(Get-Date -Format yy.MM.dd)")) +$script_content.Add($(Get-Content "scripts\start.ps1").replace('#{replaceme}',"$(Get-Date -Format yy.MM.dd)")) Update-Progress "Adding: Functions" 20 -Get-ChildItem "$workingdir\functions" -Recurse -File | ForEach-Object { +Get-ChildItem "functions" -Recurse -File | ForEach-Object { $script_content.Add($(Get-Content $psitem.FullName)) } Update-Progress "Adding: Config *.json" 40 -Get-ChildItem "$workingdir\config" | Where-Object {$psitem.extension -eq ".json"} | ForEach-Object { +Get-ChildItem "config" | Where-Object {$psitem.extension -eq ".json"} | ForEach-Object { $json = (Get-Content $psitem.FullName).replace("'","''") $jsonAsObject = $json | convertfrom-json @@ -85,34 +88,33 @@ Get-ChildItem "$workingdir\config" | Where-Object {$psitem.extension -eq ".json" $script_content.Add($(Write-output "`$sync.configs.$($psitem.BaseName) = '$json' `| convertfrom-json" )) } -$xaml = (Get-Content "$workingdir\xaml\inputXML.xaml").replace("'","''") +$xaml = (Get-Content "xaml\inputXML.xaml").replace("'","''") Update-Progress "Adding: Xaml " 90 $script_content.Add($(Write-output "`$inputXML = '$xaml'")) -$script_content.Add($(Get-Content "$workingdir\scripts\main.ps1")) +$script_content.Add($(Get-Content "scripts\main.ps1")) if ($Debug) { Update-Progress "Writing debug files" 95 - $appXamlContent | Out-File -FilePath "$workingdir\xaml\inputApp.xaml" -Encoding ascii - $tweaksXamlContent | Out-File -FilePath "$workingdir\xaml\inputTweaks.xaml" -Encoding ascii - $featuresXamlContent | Out-File -FilePath "$workingdir\xaml\inputFeatures.xaml" -Encoding ascii + $appXamlContent | Out-File -FilePath "xaml\inputApp.xaml" -Encoding ascii + $tweaksXamlContent | Out-File -FilePath "xaml\inputTweaks.xaml" -Encoding ascii + $featuresXamlContent | Out-File -FilePath "xaml\inputFeatures.xaml" -Encoding ascii } else { Update-Progress "Removing temporary files" 99 - Remove-Item "$workingdir\xaml\inputApp.xaml" -ErrorAction SilentlyContinue - Remove-Item "$workingdir\xaml\inputTweaks.xaml" -ErrorAction SilentlyContinue - Remove-Item "$workingdir\xaml\inputFeatures.xaml" -ErrorAction SilentlyContinue + Remove-Item "xaml\inputApp.xaml" -ErrorAction SilentlyContinue + Remove-Item "xaml\inputTweaks.xaml" -ErrorAction SilentlyContinue + Remove-Item "xaml\inputFeatures.xaml" -ErrorAction SilentlyContinue } -Set-Content -Path "$workingdir\$scriptname" -Value ($script_content -join "`r`n") -Encoding ascii +Set-Content -Path "$scriptname" -Value ($script_content -join "`r`n") -Encoding ascii Write-Progress -Activity "Compiling" -Completed Update-Progress -Activity "Validating" -StatusMessage "Checking winutil.ps1 Syntax" -Percent 0 try { $null = Get-Command -Syntax .\winutil.ps1 -} -catch { +} catch { Write-Warning "Syntax Validation for 'winutil.ps1' has failed" Write-Host "$($Error[0])" -ForegroundColor Red } @@ -128,3 +130,4 @@ if ($run) { break } +Pop-Location diff --git a/config/autounattend.xml b/config/autounattend.xml index c3a2f2138c..6a432641b9 100644 --- a/config/autounattend.xml +++ b/config/autounattend.xml @@ -293,7 +293,7 @@ param( - [xml] $Document + [xml]$Document ); $scriptsDir = 'C:\Windows\Setup\Scripts\'; diff --git a/functions/private/Invoke-WinUtilGPU.ps1 b/functions/private/Invoke-WinUtilGPU.ps1 index 2bcbb01b00..6ec42d7afe 100644 --- a/functions/private/Invoke-WinUtilGPU.ps1 +++ b/functions/private/Invoke-WinUtilGPU.ps1 @@ -20,4 +20,4 @@ function Invoke-WinUtilGPU { } } return $true -} \ No newline at end of file +} diff --git a/functions/private/Invoke-WinUtilMicroWin-Helper.ps1 b/functions/private/Invoke-WinUtilMicroWin-Helper.ps1 index d1061d67f3..5590d501af 100644 --- a/functions/private/Invoke-WinUtilMicroWin-Helper.ps1 +++ b/functions/private/Invoke-WinUtilMicroWin-Helper.ps1 @@ -259,7 +259,7 @@ function New-Unattend { param ( [Parameter(Mandatory, Position = 0)] [string]$userName, - [Parameter(Position = 1)] [string] $userPassword + [Parameter(Position = 1)] [string]$userPassword ) $unattend = @' diff --git a/scripts/main.ps1 b/scripts/main.ps1 index 0d706c3259..bcc209b2d6 100644 --- a/scripts/main.ps1 +++ b/scripts/main.ps1 @@ -32,19 +32,19 @@ $sync.runspace.Open() # Create classes for different exceptions class WingetFailedInstall : Exception { - [string] $additionalData + [string]$additionalData WingetFailedInstall($Message) : base($Message) {} } class ChocoFailedInstall : Exception { - [string] $additionalData + [string]$additionalData ChocoFailedInstall($Message) : base($Message) {} } class GenericException : Exception { - [string] $additionalData + [string]$additionalData GenericException($Message) : base($Message) {} } diff --git a/tools/Invoke-Preprocessing.ps1 b/tools/Invoke-Preprocessing.ps1 index c9692fc68f..96b919d197 100644 --- a/tools/Invoke-Preprocessing.ps1 +++ b/tools/Invoke-Preprocessing.ps1 @@ -39,7 +39,7 @@ .EXAMPLE Invoke-Preprocessing -ThrowExceptionOnEmptyFilesList -WorkingDir "DRIVE:\Path\To\Folder\" -ExcludedFiles @('file.txt', '.\.git\', '*.png') -ProgressStatusMessage "Doing Preprocessing" - Same as Example No. 1, but will throw an exception when 'Invoke-Preprocessing' function doesn't find any files in 'WorkingDir' (not including 'ExcludedFiles' list). + Same as Example No. 1, but uses '-ThrowExceptionOnEmptyFilesList', which's an optional parameter that'll make 'Invoke-Preprocessing' throw an exception when no files are found in 'WorkingDir' (not including the ExcludedFiles, of course), useful when you want to double check your parameters & you're sure there's files to process in the 'WorkingDir'. .EXAMPLE Invoke-Preprocessing -Skip -WorkingDir "DRIVE:\Path\To\Folder\" -ExcludedFiles @('file.txt', '.\.git\', '*.png') -ProgressStatusMessage "Doing Preprocessing" @@ -47,7 +47,7 @@ Same as Example No. 1, but uses '-SkipExcludedFilesValidation', which'll skip the validation step for 'ExcludedFiles' list. This can be useful when 'ExcludedFiles' list is generated from another function, or from unreliable source (you can't guarantee every item in list is a valid path), but you want to silently continue through the function. #> - param ( + param ( [Parameter(position=0)] [switch]$SkipExcludedFilesValidation, @@ -66,28 +66,86 @@ [Parameter(position=5)] [string]$ProgressActivity = "Preprocessing" - ) + ) if (-NOT (Test-Path -PathType Container -Path "$WorkingDir")) { throw "[Invoke-Preprocessing] Invalid Paramter Value for 'WorkingDir', passed value: '$WorkingDir'. Either the path is a File or Non-Existing/Invlid, please double check your code." } $count = $ExcludedFiles.Count - if ((-NOT ($count -eq 0)) -AND (-NOT $SkipExcludedFilesValidation)) { + + # Make sure there's a * at the end of folders in ExcludedFiles list + for ($i = 0; $i -lt $count; $i++) { + $excludedFile = $ExcludedFiles[$i] + $isFolder = ($excludedFile) -match '\\$' + if ($isFolder) { $ExcludedFiles[$i] = $excludedFile + '*' } + } + + # Validate the ExcludedFiles List before continuing on, + # that's if there's a list in the first place, and '-SkipExcludedFilesValidation' was not provided. + if (-not $SkipExcludedFilesValidation) { for ($i = 0; $i -lt $count; $i++) { $excludedFile = $ExcludedFiles[$i] $filePath = "$(($WorkingDir -replace ('\\$', '')) + '\' + ($excludedFile -replace ('\.\\', '')))" - if (-NOT (Get-ChildItem -Recurse -Path "$filePath" -File)) { - $failedFilesList += "'$filePath', " + + # Handle paths with wildcards in a different implementation + $matches = ($filePath) -match '^.*?\*' + + if ($matches) { + if (-NOT (Get-ChildItem -Recurse -Path "$filePath" -File)) { + $failedFilesList += "'$filePath', " + } + } else { + if (-NOT (Test-Path -Path "$filePath")) { + $failedFilesList += "'$filePath', " + } } } $failedFilesList = $failedFilesList -replace (',\s*$', '') if (-NOT $failedFilesList -eq "") { - throw "[Invoke-Preprocessing] One or more File Paths & File Patterns were not found, you can use '-SkipExcludedFilesValidation' switch to skip this check, and the failed files are: $failedFilesList" + throw "[Invoke-Preprocessing] One or more File Paths and/or File Patterns were not found, you can use '-SkipExcludedFilesValidation' switch to skip this check, the failed to validate are: $failedFilesList" + } + } + + # Get Files List + [System.Collections.ArrayList]$files = Get-ChildItem $WorkingDir -Recurse -Exclude $ExcludedFiles -File + $numOfFiles = $files.Count + + # Only keep the 'FullName' Property for every entry in the list + for ($i = 0; $i -lt $numOfFiles; $i++) { + $file = $files[$i] + $files[$i] = $file.FullName + } + + # If a file(s) are found in Exclude List, + # Remove the file from files list. + for ($j = 0; $j -lt $excludedFiles.Count; $j++) { + # Prepare some variables + $excluded = $excludedFiles[$j] + $pathToFind = ($excluded) -replace ('^\.\\', '') + $pathToFind = $WorkingDir + '\' + $pathToFind + $index = -1 # reset index on every iteration + + # Handle paths with wildcards in a different implementation + $matches = ($pathToFind) -match '^.*?\*' + + if ($matches) { + $filesToCheck = Get-ChildItem -Recurse -Path "$pathToFind" -File + if ($filesToCheck) { + for ($k = 0; $k -lt $filesToCheck.Count; $k++) { + $fileToCheck = $filesToCheck[$k] + $index = $files.IndexOf("$fileToCheck") + if ($index -ge 0) { $files.RemoveAt($index) } + } + } + } else { + $index = $files.IndexOf("$pathToFind") + if ($index -ge 0) { $files.RemoveAt($index) } } } - $files = Get-ChildItem $WorkingDir -Recurse -Exclude $ExcludedFiles -File + # Make sure 'numOfFiles' is synced with the actual Number of Files found in '$files' + # This's done because previous may or may not edit the files list, so we should update it $numOfFiles = $files.Count if ($numOfFiles -eq 0) { @@ -99,26 +157,11 @@ } for ($i = 0; $i -lt $numOfFiles; $i++) { - $file = $files[$i] - - # If the file is in Exclude List, don't proceed to check/modify said file. - $fileIsExcluded = $False - for ($j = 0; $j -lt $excludedFiles.Count; $j++) { - $excluded = $excludedFiles[$j] - $strToCompare = ($excluded) -replace ('^\.\\', '') - if ($file.FullName.Contains("$strToCompare")) { - $fileIsExcluded = $True - break - } - } - - if ($fileIsExcluded) { - continue - } + $fullFileName = $files[$i] # TODO: # make more formatting rules, and document them in WinUtil Official Documentation - (Get-Content "$file").TrimEnd() ` + (Get-Content "$fullFileName").TrimEnd() ` -replace ('\t', ' ') ` -replace ('\)\s*\{', ') {') ` -replace ('(?if|for|foreach)\s*(?\([.*?]\))\s*\{', '${keyword} ${condition} {') ` @@ -129,8 +172,8 @@ -replace ('\}\s*Catch', '} catch') ` -replace ('\}\s*Catch\s*(?(\[.*?\]\s*(\,)?\s*)+)\s*\{', '} catch ${exceptions} {') ` -replace ('\}\s*Catch\s*(?\[.*?\])\s*\{', '} catch ${exceptions} {') ` - -replace ('(?\[.*?\])\s*(?\$.*?(,|\s*\)))', '${parameter_type}${str_after_type}') ` - | Set-Content "$file" + -replace ('(?\[[^$0-9]+\])\s*(?\$.*?)', '${parameter_type}${str_after_type}') ` + | Set-Content "$fullFileName" Write-Progress -Activity $ProgressActivity -Status "$ProgressStatusMessage - Finished $i out of $numOfFiles" -PercentComplete (($i/$numOfFiles)*100) }