From a66a15b291f5e85a108d8528cd2efdb412ed7ac3 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 27 Oct 2021 17:15:28 -0700 Subject: [PATCH 01/38] Move mapping file into product repo and add Debian 11 (#16316) --- tools/packages.microsoft.com/mapping.json | 25 +++++++++++++++++++ .../release-PublishPackageMsftCom.yml | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 tools/packages.microsoft.com/mapping.json diff --git a/tools/packages.microsoft.com/mapping.json b/tools/packages.microsoft.com/mapping.json new file mode 100644 index 00000000000..e7d520a97de --- /dev/null +++ b/tools/packages.microsoft.com/mapping.json @@ -0,0 +1,25 @@ +{ + "Packages" : [ + { + "url": "microsoft-rhel7.3-prod", + "distribution" : ["trusty"], + "PackageFormat" : "PACKAGE_NAME-POWERSHELL_RELEASE-1.rhel.7.x86_64.rpm" + }, + { + "distribution" : ["stretch"], + "PackageFormat" : "PACKAGE_NAME_POWERSHELL_RELEASE-1.debian.9_amd64.deb" + }, + { + "distribution" : ["buster"], + "PackageFormat" : "PACKAGE_NAME_POWERSHELL_RELEASE-1.debian.10_amd64.deb" + }, + { + "distribution" : ["bionic"], + "PackageFormat" : "PACKAGE_NAME_POWERSHELL_RELEASE-1.ubuntu.18.04_amd64.deb" + }, + { + "distribution": ["xenial"], + "PackageFormat": "PACKAGE_NAME_POWERSHELL_RELEASE-1.ubuntu.16.04_amd64.deb" + } + ] +} diff --git a/tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml b/tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml index cb5ba23f5b9..ca1c7facfd8 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml @@ -34,5 +34,5 @@ steps: LTS = $metadata.LTSRelease ForProduction = $true } - & '$(Pipeline.Workspace)/tools/packages.microsoft.com/releaseLinuxPackages.ps1' @params + & '$(Pipeline.Workspace)/tools/packages.microsoft.com/releaseLinuxPackages.ps1' -MappingFilePath '$(System.DefaultWorkingDirectory)/tools/packages.microsoft.com/mapping.json' @params displayName: Run release script From 13b8a0f823c9f38089b3e50355a9c39962ce2cc2 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Mon, 8 Nov 2021 12:28:09 -0800 Subject: [PATCH 02/38] Add checkout to build json stage to get `ci.psm1` (#16399) --- tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml b/tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml index 52fae08e8b9..f03e8f25d7c 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml @@ -1,4 +1,7 @@ steps: +- checkout: self + clean: true + - task: DownloadPipelineArtifact@2 inputs: source: specific From 344986bac01b56f9d23bf28299d042f0ee9d5706 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Mon, 13 Dec 2021 13:57:07 -0800 Subject: [PATCH 03/38] Update macOS build image and root folder for build (#16609) # Conflicts: # tools/releaseBuild/azureDevOps/templates/mac.yml --- .../azureDevOps/templates/cloneToOfficialPath.yml | 2 +- .../azureDevOps/templates/mac-package-build.yml | 2 +- tools/releaseBuild/azureDevOps/templates/mac.yml | 12 +++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml index 0a52f887671..4985433aed0 100644 --- a/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml +++ b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml @@ -1,7 +1,7 @@ steps: - powershell: | $dirSeparatorChar = [system.io.path]::DirectorySeparatorChar - $nativePath = "${dirSeparatorChar}PowerShell" + $nativePath = "${env:HOME}${dirSeparatorChar}PowerShell" Write-Host "##vso[task.setvariable variable=PowerShellRoot]$nativePath" if ((Test-Path "$nativePath")) { diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index a5b9ad9f7c2..a7a42f2f1d1 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -8,7 +8,7 @@ jobs: dependsOn: ${{ parameters.parentJob }} condition: succeeded() pool: - vmImage: internal-macos-10.14 + vmImage: internal-macos-11 variables: # Turn off Homebrew analytics - name: HOMEBREW_NO_ANALYTICS diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index eb8ea9d0688..f4095e0176f 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -6,7 +6,7 @@ jobs: displayName: Build macOS condition: succeeded() pool: - vmImage: internal-macos-10.14 + vmImage: internal-macos-11 variables: # Turn off Homebrew analytics - name: HOMEBREW_NO_ANALYTICS @@ -28,11 +28,11 @@ jobs: - pwsh: | # create folder - sudo mkdir /PowerShell + sudo mkdir "${env:HOME}/PowerShell" # make the current user the owner - sudo chown $env:USER /PowerShell - displayName: 'Create /PowerShell' + sudo chown $env:USER "${env:HOME}/PowerShell" + displayName: 'Create ${env:HOME}/PowerShell' - template: cloneToOfficialPath.yml @@ -46,7 +46,9 @@ jobs: - pwsh: | $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults + # Add -SkipReleaseChecks as a mitigation to unblock release. + # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. + $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks $env:AzDevOpsFeedPAT2 = $null displayName: 'Build' From 2c9a6b891e7f5fcca8c236b21ab59e4a66332618 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Mon, 13 Dec 2021 16:13:17 -0800 Subject: [PATCH 04/38] Update the `vmImage` and PowerShell root directory for macOS builds (#16611) # Conflicts: # tools/releaseBuild/azureDevOps/templates/mac-package-build.yml --- .../templates/cloneToOfficialPath.yml | 5 ++++- .../templates/mac-package-build.yml | 22 +++++++++---------- .../azureDevOps/templates/mac.yml | 10 +++++---- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml index 4985433aed0..352458390f9 100644 --- a/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml +++ b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml @@ -1,7 +1,10 @@ +parameters: + nativePathRoot: '' + steps: - powershell: | $dirSeparatorChar = [system.io.path]::DirectorySeparatorChar - $nativePath = "${env:HOME}${dirSeparatorChar}PowerShell" + $nativePath = "${{parameters.nativePathRoot }}${dirSeparatorChar}PowerShell" Write-Host "##vso[task.setvariable variable=PowerShellRoot]$nativePath" if ((Test-Path "$nativePath")) { diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index a7a42f2f1d1..a276302e466 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -8,7 +8,7 @@ jobs: dependsOn: ${{ parameters.parentJob }} condition: succeeded() pool: - vmImage: internal-macos-11 + vmImage: macos-latest variables: # Turn off Homebrew analytics - name: HOMEBREW_NO_ANALYTICS @@ -23,23 +23,21 @@ jobs: - pwsh: | # create folder - sudo mkdir /PowerShell + sudo mkdir "$(Agent.TempDirectory)/PowerShell" # make the current user the owner - sudo chown $env:USER /PowerShell - displayName: 'Create /PowerShell' + sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell" + displayName: 'Create $(Agent.TempDirectory)/PowerShell' - - template: shouldSign.yml - - #- task: @ - # inputs: - # - # displayName: '' - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) + - template: shouldSign.yml + - template: cloneToOfficialPath.yml + parameters: + nativePathRoot: '$(Agent.TempDirectory)' - task: DownloadBuildArtifacts@0 displayName: Download macosBinResults @@ -107,7 +105,9 @@ jobs: displayName: 'Bootstrap VM' - pwsh: | - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" + # Add -SkipReleaseChecks as a mitigation to unblock release. + # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. + $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" -SkipReleaseChecks displayName: 'Package' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index f4095e0176f..220b08c4f22 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -6,7 +6,7 @@ jobs: displayName: Build macOS condition: succeeded() pool: - vmImage: internal-macos-11 + vmImage: macos-latest variables: # Turn off Homebrew analytics - name: HOMEBREW_NO_ANALYTICS @@ -28,13 +28,15 @@ jobs: - pwsh: | # create folder - sudo mkdir "${env:HOME}/PowerShell" + sudo mkdir "$(Agent.TempDirectory)/PowerShell" # make the current user the owner - sudo chown $env:USER "${env:HOME}/PowerShell" - displayName: 'Create ${env:HOME}/PowerShell' + sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell" + displayName: 'Create $(Agent.TempDirectory)/PowerShell' - template: cloneToOfficialPath.yml + parameters: + nativePathRoot: '$(Agent.TempDirectory)' - pwsh: | tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap From 899102c78256a4d86a3bc7fdbac98b163d74e506 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 28 Oct 2021 11:44:06 -0700 Subject: [PATCH 05/38] Add GitHub Workflow to keep notices up to date (#16284) Co-authored-by: Robert Holt # Conflicts: # build.psm1 # tools/findMissingNotices.ps1 # Conflicts: # build.psm1 --- .github/workflows/update-cgmanifest.yml | 64 ++++++ build.psm1 | 49 +---- tools/buildCommon/startNativeExecution.ps1 | 47 +++++ tools/findMissingNotices.ps1 | 220 +++++++++++++++++++++ 4 files changed, 333 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/update-cgmanifest.yml create mode 100644 tools/buildCommon/startNativeExecution.ps1 create mode 100644 tools/findMissingNotices.ps1 diff --git a/.github/workflows/update-cgmanifest.yml b/.github/workflows/update-cgmanifest.yml new file mode 100644 index 00000000000..9e026b70f86 --- /dev/null +++ b/.github/workflows/update-cgmanifest.yml @@ -0,0 +1,64 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +name: Update cgmanifest +on: + workflow_dispatch: + schedule: + # At 13:00 UTC every day. + - cron: '0 13 * * *' + +defaults: + run: + shell: pwsh + +env: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + POWERSHELL_TELEMETRY_OPTOUT: 1 + +jobs: + update-cgmanifest: + name: Update cgmanifest + timeout-minutes: 15 + runs-on: windows-latest + if: github.repository == 'PowerShell/PowerShell' + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Sync tags + run: | + git fetch --prune --unshallow --tags + - name: Install Ships provider to deal with project.assets.json + run: | + Install-Module -Name dotnet.project.assets -force + - name: Bootstrap + run: | + Import-Module ./build.psm1 + Start-PSBootStrap + - name: Update Notices file + run: | + Invoke-WebRequest -Uri https://aka.ms/pwsh-daily-tpn -OutFile ./ThirdPartyNotices.txt + - name: Execute script to update cgmanifest + run: | + Import-Module ./build.psm1 + Find-Dotnet + ./tools/findMissingNotices.ps1 + - name: Microsoft Teams Notifier + uses: skitionek/notify-microsoft-teams@master + if: failure() + with: + webhook_url: ${{ secrets.PS_BUILD_TEAMS_CHANNEL }} + overwrite: "{title: `Failure in updating cgmanifest. Look at ${workflow_link}`}" + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + id: cpr + if: env.CREATE_PR == 'true' + with: + commit-message: "Update the cgmanifest with missing or updated components" + committer: GitHub + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + title: "Update ${{ env.FORMULA_NAME }} formula to version ${{ env.NEW_FORMULA_VERSION }}" + reviewers: travisez13 + base: master + draft: false + branch: update-cgmanifest diff --git a/build.psm1 b/build.psm1 index 02c4ca14ed1..06bfd9be6cd 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +. "$PSScriptRoot\tools\buildCommon\startNativeExecution.ps1" + # On Unix paths is separated by colon # On Windows paths is separated by semicolon $script:TestModulePathSeparator = [System.IO.Path]::PathSeparator @@ -2136,53 +2138,6 @@ function script:precheck([string]$command, [string]$missedMessage) { } } -# this function wraps native command Execution -# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/ -function script:Start-NativeExecution -{ - param( - [scriptblock]$sb, - [switch]$IgnoreExitcode, - [switch]$VerboseOutputOnError - ) - $backupEAP = $script:ErrorActionPreference - $script:ErrorActionPreference = "Continue" - try { - if($VerboseOutputOnError.IsPresent) - { - $output = & $sb 2>&1 - } - else - { - & $sb - } - - # note, if $sb doesn't have a native invocation, $LASTEXITCODE will - # point to the obsolete value - if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) { - if($VerboseOutputOnError.IsPresent -and $output) - { - $output | Out-String | Write-Verbose -Verbose - } - - # Get caller location for easier debugging - $caller = Get-PSCallStack -ErrorAction SilentlyContinue - if($caller) - { - $callerLocationParts = $caller[1].Location -split ":\s*line\s*" - $callerFile = $callerLocationParts[0] - $callerLine = $callerLocationParts[1] - - $errorMessage = "Execution of {$sb} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE" - throw $errorMessage - } - throw "Execution of {$sb} failed with exit code $LASTEXITCODE" - } - } finally { - $script:ErrorActionPreference = $backupEAP - } -} - function Start-CrossGen { [CmdletBinding()] param( diff --git a/tools/buildCommon/startNativeExecution.ps1 b/tools/buildCommon/startNativeExecution.ps1 new file mode 100644 index 00000000000..ee7b00d04cd --- /dev/null +++ b/tools/buildCommon/startNativeExecution.ps1 @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# this function wraps native command Execution +# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/ +function script:Start-NativeExecution { + param( + [Alias('sb')] + [Parameter(Mandatory=$true)] + [scriptblock]$ScriptBlock, + [switch]$IgnoreExitcode, + [switch]$VerboseOutputOnError + ) + + $backupEAP = $ErrorActionPreference + $ErrorActionPreference = "Continue" + Write-Verbose "Executing: $ScriptBlock" + try { + if ($VerboseOutputOnError.IsPresent) { + $output = & $ScriptBlock 2>&1 + } else { + & $ScriptBlock + } + + # note, if $ScriptBlock doesn't have a native invocation, $LASTEXITCODE will + # point to the obsolete value + if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) { + if ($VerboseOutputOnError.IsPresent -and $output) { + $output | Out-String | Write-Verbose -Verbose + } + + # Get caller location for easier debugging + $caller = Get-PSCallStack -ErrorAction SilentlyContinue + if ($caller) { + $callerLocationParts = $caller[1].Location -split ":\s*line\s*" + $callerFile = $callerLocationParts[0] + $callerLine = $callerLocationParts[1] + + $errorMessage = "Execution of {$ScriptBlock} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE" + throw $errorMessage + } + throw "Execution of {$ScriptBlock} failed with exit code $LASTEXITCODE" + } + } finally { + $ErrorActionPreference = $backupEAP + } +} diff --git a/tools/findMissingNotices.ps1 b/tools/findMissingNotices.ps1 new file mode 100644 index 00000000000..f2c8066234a --- /dev/null +++ b/tools/findMissingNotices.ps1 @@ -0,0 +1,220 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# This script is used to completely rebuild the cgmanifgest.json file, +# which is used to generate the notice file. +# Requires the module dotnet.project.assets from the PowerShell Gallery authored by @TravisEz13 + +Import-Module dotnet.project.assets +. "$PSScriptRoot\..\tools\buildCommon\startNativeExecution.ps1" + +$existingRegistrationTable = @{} +$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable +$existingRegistrationsJson.Registrations | ForEach-Object { + $registration = [Registration]$_ + if($registration.Component) { + $name = $registration.Component.Name() + $existingRegistrationTable.Add($name, $registration) + } +} + +Class Registration { + [Component]$Component + [bool]$DevelopmentDependency +} + +Class Component { + [ValidateSet("nuget")] + [String] $Type + [Nuget]$Nuget + + [string]ToString() { + $message = "Type: $($this.Type)" + if ($this.Type -eq "nuget") { + $message += "; $($this.Nuget)" + } + return $message + } + + [string]Name() { + switch ($this.Type) { + "nuget" { + return $($this.Nuget.Name) + } + default { + throw "Unknown component type: $($this.Type)" + } + } + throw "How did we get here?!?" + } + + [string]Version() { + switch ($this.Type) { + "nuget" { + return $($this.Nuget.Version) + } + default { + throw "Unknown component type: $($this.Type)" + } + } + throw "How did we get here?!?" + } +} + +Class Nuget { + [string]$Name + [string]$Version + + [string]ToString() { + return "$($this.Name) - $($this.Version)" + } +} + +function New-NugetComponent { + param( + [string]$name, + [string]$version + ) + + $nuget = [Nuget]@{ + Name = $name + Version = $version + } + $Component = [Component]@{ + Type = "nuget" + Nuget = $nuget + } + + $registration = [Registration]@{ + Component = $Component + DevelopmentDependency = $false + } + + return $registration +} + +$winDesktopSdk = 'Microsoft.NET.Sdk.WindowsDesktop' +if (!$IsWindows) { + $winDesktopSdk = 'Microsoft.NET.Sdk' + Write-Warning "Always using $winDesktopSdk since this is not windows!!!" +} + +Function Get-CGRegistrations { + param( + [Parameter(Mandatory)] + [ValidateSet( + "alpine-x64", + "linux-arm", + "linux-arm64", + "linux-x64", + "osx-arm64", + "osx-x64", + "win-arm", + "win-arm64", + "win7-x64", + "win7-x86", + "modules")] + [string]$Runtime, + + [Parameter(Mandatory)] + [System.Collections.Generic.Dictionary[string, Registration]] $RegistrationTable + ) + + $newRegistrations = $Registrations + + $dotnetTargetName = 'net6.0' + $dotnetTargetNameWin7 = 'net6.0-windows7.0' + $unixProjectName = 'powershell-unix' + $windowsProjectName = 'powershell-win-core' + $actualRuntime = $Runtime + + switch -regex ($Runtime) { + "alpine-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "linux-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "osx-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "win7-.*" { + $sdkToUse = $winDesktopSdk + $folder = $windowsProjectName + $target = "$dotnetTargetNameWin7|$Runtime" + } + "win-.*" { + $folder = $windowsProjectName + $target = "$dotnetTargetNameWin7|$Runtime" + } + "modules" { + $folder = "modules" + $actualRuntime = 'linux-x64' + $target = "$dotnetTargetName|$actualRuntime" + } + Default { + throw "Invalid runtime name: $Runtime" + } + } + + Write-Verbose "Getting registrations for $folder - $actualRuntime ..." -Verbose + Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive + Push-Location $PSScriptRoot\..\src\$folder + try { + Start-NativeExecution -VerboseOutputOnError -sb { + dotnet restore --runtime $actualRuntime "/property:SDKToUse=$sdkToUse" + } + $null = New-PADrive -Path $PSScriptRoot\..\src\$folder\obj\project.assets.json -Name $folder + try { + $targets = Get-ChildItem -Path "${folder}:/targets/$target" -ErrorAction Stop | Where-Object { + $_.Type -eq 'package' -and + $_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and + $_.Name -notlike 'StyleCop*' -and + $_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and + $_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*' + } | select-object -ExpandProperty name + } catch { + Get-ChildItem -Path "${folder}:/targets" | Out-String | Write-Verbose -Verbose + throw + } + } finally { + Pop-Location + Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive + } + + $targets | ForEach-Object { + $target = $_ + $parts = ($target -split '\|') + $name = $parts[0] + $targetVersion = $parts[1] + $pattern = [regex]::Escape($name) + " " + $tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern + + # Add the registration to the cgmanifest if the TPN does not contain the name of the target OR + # the exisitng CG contains the registration, because if the existing CG contains the registration, + # that might be the only reason it is in the TPN. + if ((!$tpnMatch -or $existingRegistrationTable.ContainsKey($name)) -and !$RegistrationTable.ContainsKey($target)) { + $registration = New-NugetComponent -Name $name -Version $targetVersion + $RegistrationTable.Add($target, $registration) + } + } +} + +$registrations = [System.Collections.Generic.Dictionary[string, Registration]]::new() +$lastCount = 0 +foreach ($runtime in "win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win-arm", "linux-arm", "linux-arm64", "osx-arm64", "win-arm64", "win7-x86") { + Get-CGRegistrations -Runtime $runtime -RegistrationTable $registrations + $count = $registrations.Count + $newCount = $count - $lastCount + $lastCount = $count + Write-Verbose "$newCount new registrations, $count total..." -Verbose +} + +$newRegistrations = $registrations.Keys | Sort-Object | ForEach-Object { $registrations[$_] } + +$count = $newRegistrations.Count +@{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 | Set-Content $PSScriptRoot\..\cgmanifest.json +Write-Verbose "$count registrations created!" -Verbose From f71203b410ac261c1436848f43b55858d12a2ab8 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 12 Oct 2021 15:40:43 -0700 Subject: [PATCH 06/38] Add Software Bill of Materials to the main packages (#16202) # Conflicts: # .vsts-ci/templates/windows-packaging.yml # .vsts-ci/windows/windows-packaging.yml # assets/files.wxs # build.psm1 # tools/ci.psm1 # tools/packaging/packaging.psd1 # tools/packaging/packaging.psm1 # Conflicts: # build.psm1 # tools/ci.psm1 # tools/packaging/packaging.psd1 # tools/packaging/packaging.psm1 --- .vsts-ci/templates/windows-packaging.yml | 55 +++- build.psm1 | 16 +- tools/WindowsCI.psm1 | 5 - tools/ci.psm1 | 186 +++++++++---- tools/packaging/packaging.psd1 | 35 ++- tools/packaging/packaging.psm1 | 259 +++++++++++++++++- tools/packaging/releaseTests/sbom.tests.ps1 | 110 ++++++++ .../releaseBuild/azureDevOps/releaseBuild.yml | 8 + .../azureDevOps/releasePipeline.yml | 17 +- .../azureDevOps/templates/linux.yml | 101 +++++-- .../templates/mac-package-build.yml | 5 + .../release-SetReleaseTagAndContainerName.yml | 4 +- .../templates/release-ValidatePackageBOM.yml | 43 +++ .../templates/windows-packaging.yml | 5 + 14 files changed, 736 insertions(+), 113 deletions(-) create mode 100644 tools/packaging/releaseTests/sbom.tests.ps1 create mode 100644 tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml diff --git a/.vsts-ci/templates/windows-packaging.yml b/.vsts-ci/templates/windows-packaging.yml index 0bfa92446e3..82219997891 100644 --- a/.vsts-ci/templates/windows-packaging.yml +++ b/.vsts-ci/templates/windows-packaging.yml @@ -1,18 +1,40 @@ parameters: - pool: 'Hosted VS2017' - jobName: 'win_packaging' - parentJobs: [] + - name: pool + default: 'Hosted VS2017' + - name: jobName + default: 'win_packaging' + - name: architecture + default: 'x64' + - name: channel + default: 'preview' jobs: -- job: ${{ parameters.jobName }} - dependsOn: - ${{ parameters.parentJobs }} +- job: ${{ parameters.jobName }}_${{ parameters.channel }}_${{ parameters.architecture }} + + variables: + - name: repoFolder + value: PowerShell + - name: repoPath + value: $(Agent.BuildDirectory)\$(repoFolder) + - name: complianceRepoFolder + value: compliance + - name: complianceRepoPath + value: $(Agent.BuildDirectory)\$(complianceRepoFolder) + pool: name: ${{ parameters.pool }} displayName: Windows Packaging steps: + - checkout: self + clean: true + path: $(repoFolder) + + - checkout: ComplianceRepo + clean: true + path: $(complianceRepoFolder) + - powershell: | Get-ChildItem -Path env: displayName: Capture environment @@ -24,10 +46,25 @@ jobs: Import-Module .\tools\ci.psm1 Invoke-CIInstall -SkipUser displayName: Bootstrap - condition: succeededOrFailed() + condition: succeeded() + workingDirectory: $(repoPath) + + - pwsh: | + Import-Module .\tools\ci.psm1 + New-CodeCoverageAndTestPackage + Invoke-CIFinish -Runtime win7-${{ parameters.architecture }} -channel ${{ parameters.channel }} -Stage Build + displayName: Build + workingDirectory: $(repoPath) + + - template: Sbom.yml@ComplianceRepo + parameters: + BuildDropPath: '$(System.ArtifactsDirectory)/mainBuild' + Build_Repository_Uri: $(build.repository.uri) + displayName: SBOM - pwsh: | Import-Module .\tools\ci.psm1 New-CodeCoverageAndTestPackage - Invoke-CIFinish -NuGetKey $(NUGET_KEY) - displayName: Build and Test Package + Invoke-CIFinish -Runtime win7-${{ parameters.architecture }} -channel ${{ parameters.channel }} -Stage Package + displayName: Package and Test + workingDirectory: $(repoPath) diff --git a/build.psm1 b/build.psm1 index 06bfd9be6cd..fa87449e6a6 100644 --- a/build.psm1 +++ b/build.psm1 @@ -296,9 +296,14 @@ function Start-PSBuild { [ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d+)?)?$")] [ValidateNotNullOrEmpty()] [string]$ReleaseTag, - [switch]$Detailed + [switch]$Detailed, + [string]$PSOptionsPath ) + if ($ReleaseTag -and $ReleaseTag -notmatch "^v\d+\.\d+\.\d+(-(preview|rc)(\.\d{1,2})?)?$") { + Write-Warning "Only preview or rc are supported for releasing pre-release version of PowerShell" + } + if ($PsCmdlet.ParameterSetName -eq "Default" -and !$NoPSModuleRestore) { $PSModuleRestore = $true @@ -589,6 +594,15 @@ Fix steps: if ($CI) { Restore-PSPester -Destination (Join-Path $publishPath "Modules") } + + if ($PSOptionsPath) { + $resolvedPSOptionsPath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($PSOptionsPath) + $parent = Split-Path -Path $resolvedPSOptionsPath + if (!(Test-Path $parent)) { + $null = New-Item -ItemType Directory -Path $parent + } + Save-PSOptions -PSOptionsPath $PSOptionsPath -Options $Options + } } function Restore-PSPackage diff --git a/tools/WindowsCI.psm1 b/tools/WindowsCI.psm1 index 7c0c02265e7..bf53ce79dce 100644 --- a/tools/WindowsCI.psm1 +++ b/tools/WindowsCI.psm1 @@ -1,11 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -if($PSVersionTable.PSEdition -ne 'Desktop') -{ - throw 'Must be run from Windows PowerShell' -} - function New-LocalUser { <# diff --git a/tools/ci.psm1 b/tools/ci.psm1 index c4df06a0704..845462b841d 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -436,89 +436,157 @@ function Get-ReleaseTag function Invoke-CIFinish { param( - [string] $NuGetKey + [string] $Runtime = 'win7-x64', + [string] $Channel = 'preview', + [Validateset('Build','Package')] + [string[]] $Stage = ('Build','Package') ) - if($PSEdition -eq 'Core' -and ($IsLinux -or $IsMacOS)) - { - return New-LinuxPackage -NugetKey $NugetKey + if ($PSEdition -eq 'Core' -and ($IsLinux -or $IsMacOS) -and $Stage -contains 'Build') { + return New-LinuxPackage } + $artifacts = New-Object System.Collections.ArrayList try { - $releaseTag = Get-ReleaseTag - - $previewVersion = $releaseTag.Split('-') - $previewPrefix = $previewVersion[0] - $previewLabel = $previewVersion[1].replace('.','') + $buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/mainBuild" + + if ($Stage -contains "Build") { + if ($Channel -eq 'preview') { + $releaseTag = Get-ReleaseTag + + $previewVersion = $releaseTag.Split('-') + $previewPrefix = $previewVersion[0] + $previewLabel = $previewVersion[1].replace('.','') + + if(Test-DailyBuild) + { + $previewLabel= "daily{0}" -f $previewLabel + } + + $prereleaseIteration = (get-date).Day + $preReleaseVersion = "$previewPrefix-$previewLabel.$prereleaseIteration" + # Build clean before backing to remove files from testing + Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime -output $buildFolder -PSOptionsPath "${buildFolder}/psoptions.json" + $options = Get-PSOptions + # Remove symbol files. + $filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb' + Write-Verbose "Removing symbol files from $filter" -Verbose + Remove-Item $filter -Force -Recurse + } + else { + $releaseTag = Get-ReleaseTag + $releaseTagParts = $releaseTag.split('.') + $preReleaseVersion = $releaseTagParts[0]+ ".9.9" + Write-Verbose "newPSReleaseTag: $preReleaseVersion" -Verbose + Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime -output $buildFolder -PSOptionsPath "${buildFolder}/psoptions.json" + $options = Get-PSOptions + # Remove symbol files. + $filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb' + Write-Verbose "Removing symbol files from $filter" -Verbose + Remove-Item $filter -Force -Recurse + } - if(Test-DailyBuild) - { - $previewLabel= "daily{0}" -f $previewLabel + # Set a variable, both in the current process and in AzDevOps for the packaging stage to get the release tag + $env:CI_FINISH_RELASETAG=$preReleaseVersion + $vstsCommandString = "vso[task.setvariable variable=CI_FINISH_RELASETAG]$preReleaseVersion" + Write-Verbose -Message "$vstsCommandString" -Verbose + Write-Host -Object "##$vstsCommandString" + + $armBuildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/releaseArm32" + + # produce win-arm and win-arm64 packages if it is a daily build + Start-PSBuild -Restore -Runtime win-arm -PSModuleRestore -Configuration 'Release' -ReleaseTag $releaseTag -output $armBuildFolder -PSOptionsPath "${armBuildFolder}-meta/psoptions.json" -Crossgen + $options = Get-PSOptions + # Remove symbol files. + $filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb' + Write-Verbose "Removing symbol files from $filter" -Verbose + Remove-Item $filter -Force -Recurse + + $armBuildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/releaseArm64" + Start-PSBuild -Restore -Runtime win-arm64 -PSModuleRestore -Configuration 'Release' -ReleaseTag $releaseTag -output $armBuildFolder -PSOptionsPath "${armBuildFolder}-meta/psoptions.json" -Crossgen + $options = Get-PSOptions + # Remove symbol files. + $filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb' + Write-Verbose "Removing symbol files from $filter" -Verbose + Remove-Item $filter -Force -Recurse } - $preReleaseVersion = "$previewPrefix-$previewLabel.$env:BUILD_BUILDID" - # Build clean before backing to remove files from testing - Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean - - # Build packages - $packages = Start-PSPackage -Type msi,nupkg,zip -ReleaseTag $preReleaseVersion -SkipReleaseChecks - - $artifacts = New-Object System.Collections.ArrayList - foreach ($package in $packages) { - if (Test-Path $package) - { - Write-Log "Package found: $package" - } - else - { - Write-Warning -Message "Package NOT found: $package" + if ($Stage -contains "Package") { + Restore-PSOptions -PSOptionsPath "${buildFolder}-meta/psoptions.json" + $preReleaseVersion = $env:CI_FINISH_RELASETAG + + # Build packages $preReleaseVersion = "$previewPrefix-$previewLabel.$prereleaseIteration" + $packages = Start-PSPackage -Type msi, nupkg, zip -ReleaseTag $preReleaseVersion -SkipReleaseChecks + + foreach ($package in $packages) { + if (Test-Path $package -ErrorAction Ignore) + { + Write-Log "Package found: $package" + } + else + { + Write-Warning -Message "Package NOT found: $package" + } + + if($package -is [string]) + { + $null = $artifacts.Add($package) + } + elseif($package -is [pscustomobject] -and $package.psobject.Properties['msi']) + { + $null = $artifacts.Add($package.msi) + $null = $artifacts.Add($package.wixpdb) + } } - if($package -is [string]) - { - $null = $artifacts.Add($package) - } - elseif($package -is [pscustomobject] -and $package.msi) - { - $null = $artifacts.Add($package.msi) - $null = $artifacts.Add($package.wixpdb) - } - } + # the packaging tests find the MSI package using env:PSMsiX64Path + $env:PSMsiX64Path = $artifacts | Where-Object { $_.EndsWith(".msi")} + $architechture = $Runtime.Split('-')[1] + $exePath = New-ExePackage -ProductVersion ($preReleaseVersion -replace '^v') -ProductTargetArchitecture $architechture -MsiLocationPath $env:PSMsiX64Path + Write-Verbose "exe Path: $exePath" -Verbose + $artifacts.Add($exePath) + $env:PSExePath = $exePath + $env:PSMsiChannel = $Channel + $env:PSMsiRuntime = $Runtime - # the packaging tests find the MSI package using env:PSMsiX64Path - $env:PSMsiX64Path = $artifacts | Where-Object { $_.EndsWith(".msi")} + # Install the latest Pester and import it + $maximumPesterVersion = '4.99' + Install-Module Pester -Force -SkipPublisherCheck -MaximumVersion $maximumPesterVersion + Import-Module Pester -Force -MaximumVersion $maximumPesterVersion - # Install the latest Pester and import it - $maximumPesterVersion = '4.99' - Install-Module Pester -Force -SkipPublisherCheck -MaximumVersion $maximumPesterVersion - Import-Module Pester -Force -MaximumVersion $maximumPesterVersion + $testResultPath = Join-Path -Path $env:TEMP -ChildPath "win-package-$channel-$runtime.xml" - # start the packaging tests and get the results - $packagingTestResult = Invoke-Pester -Script (Join-Path $repoRoot '.\test\packaging\windows\') -PassThru + # start the packaging tests and get the results + $packagingTestResult = Invoke-Pester -Script (Join-Path $repoRoot '.\test\packaging\windows\') -PassThru -OutputFormat NUnitXml -OutputFile $testResultPath - # fail the CI job if the tests failed, or nothing passed - if($packagingTestResult.FailedCount -ne 0 -or !$packagingTestResult.PassedCount) - { - throw "Packaging tests failed ($($packagingTestResult.FailedCount) failed/$($packagingTestResult.PassedCount) passed)" - } + Publish-TestResults -Title "win-package-$channel-$runtime" -Path $testResultPath - # only publish assembly nuget packages if it is a daily build and tests passed - if(Test-DailyBuild) - { - $nugetArtifacts = Get-ChildItem $PSScriptRoot\packaging\nugetOutput -ErrorAction SilentlyContinue -Filter *.nupkg | Select-Object -ExpandProperty FullName - if($nugetArtifacts) + # fail the CI job if the tests failed, or nothing passed + if(-not $packagingTestResult -is [pscustomobject] -or $packagingTestResult.FailedCount -ne 0 -or $packagingTestResult.PassedCount -eq 0) + { + throw "Packaging tests failed ($($packagingTestResult.FailedCount) failed/$($packagingTestResult.PassedCount) passed)" + } + + # only publish assembly nuget packages if it is a daily build and tests passed + if(Test-DailyBuild) { - $artifacts.AddRange(@($nugetArtifacts)) + $nugetArtifacts = Get-ChildItem $PSScriptRoot\packaging\nugetOutput -ErrorAction SilentlyContinue -Filter *.nupkg | Select-Object -ExpandProperty FullName + if($nugetArtifacts) + { + $artifacts.AddRange(@($nugetArtifacts)) + } } } # produce win-arm and win-arm64 packages if it is a daily build - Start-PSBuild -Restore -Runtime win-arm -PSModuleRestore -Configuration 'Release' -ReleaseTag $releaseTag + $armBuildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/releaseArm32" + Restore-PSOptions -PSOptionsPath "${armBuildFolder}-meta/psoptions.json" $arm32Package = Start-PSPackage -Type zip -WindowsRuntime win-arm -ReleaseTag $releaseTag -SkipReleaseChecks $artifacts.Add($arm32Package) - Start-PSBuild -Restore -Runtime win-arm64 -PSModuleRestore -Configuration 'Release' -ReleaseTag $releaseTag + $armBuildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/releaseArm64" + Restore-PSOptions -PSOptionsPath "${armBuildFolder}-meta/psoptions.json" $arm64Package = Start-PSPackage -Type zip -WindowsRuntime win-arm64 -ReleaseTag $releaseTag -SkipReleaseChecks $artifacts.Add($arm64Package) diff --git a/tools/packaging/packaging.psd1 b/tools/packaging/packaging.psd1 index ebe76763724..822bccbc137 100644 --- a/tools/packaging/packaging.psd1 +++ b/tools/packaging/packaging.psd1 @@ -1,12 +1,29 @@ @{ -GUID="41857994-4283-4757-a932-0b0edb104913" -Author="PowerShell" -CompanyName="Microsoft Corporation" + GUID = "41857994-4283-4757-a932-0b0edb104913" + Author = "PowerShell" + CompanyName = "Microsoft Corporation" Copyright="Copyright (c) Microsoft Corporation. All rights reserved." -ModuleVersion="1.0.0" -PowerShellVersion="5.0" -CmdletsToExport=@() -FunctionsToExport=@('Start-PSPackage','New-PSSignedBuildZip', 'New-PSBuildZip', 'New-MSIPatch', 'Expand-PSSignedBuild', 'Publish-NugetToMyGet', 'New-DotnetSdkContainerFxdPackage', 'New-GlobalToolNupkg', 'New-ILNugetPackage', 'Update-PSSignedBuildFolder') -RootModule="packaging.psm1" -RequiredModules = @("build") + ModuleVersion = "1.0.0" + PowerShellVersion = "5.0" + CmdletsToExport = @() + FunctionsToExport = @( + 'Compress-ExePackageEngine' + 'Expand-ExePackageEngine' + 'Expand-PSSignedBuild' + 'Invoke-AzDevOpsLinuxPackageBuild' + 'Invoke-AzDevOpsLinuxPackageCreation' + 'New-DotnetSdkContainerFxdPackage' + 'New-ExePackage' + 'New-GlobalToolNupkg' + 'New-ILNugetPackage' + 'New-MSIPatch' + 'New-PSBuildZip' + 'New-PSSignedBuildZip' + 'Publish-NugetToMyGet' + 'Start-PSPackage' + 'Test-PackageManifest' + 'Update-PSSignedBuildFolder' + ) + RootModule = "packaging.psm1" + RequiredModules = @("build") } diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 665d5f7a40d..86809aa8719 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -68,11 +68,11 @@ function Start-PSPackage { # Runtime and Configuration settings required by the package ($Runtime, $Configuration) = if ($WindowsRuntime) { $WindowsRuntime, "Release" - } elseif ($Type -eq "tar-alpine") { + } elseif ($Type.Count -eq 1 -and $Type[0] -eq "tar-alpine") { New-PSOptions -Configuration "Release" -Runtime "alpine-x64" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } - } elseif ($Type -eq "tar-arm") { + } elseif ($Type.Count -eq 1 -and $Type[0] -eq "tar-arm") { New-PSOptions -Configuration "Release" -Runtime "Linux-ARM" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } - } elseif ($Type -eq "tar-arm64") { + } elseif ($Type.Count -eq 1 -and $Type[0] -eq "tar-arm64") { New-PSOptions -Configuration "Release" -Runtime "Linux-ARM64" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } } else { New-PSOptions -Configuration "Release" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } @@ -90,10 +90,10 @@ function Start-PSPackage { if ($Type -eq 'fxdependent') { $NameSuffix = "win-fxdependent" - Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration'" + Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration', Runtime: '$Runtime'" } elseif ($Type -eq 'fxdependent-win-desktop') { $NameSuffix = "win-fxdependentWinDesktop" - Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration'" + Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration', Runtime: '$Runtime'" } else { Write-Log "Packaging RID: '$Runtime'; Packaging Configuration: '$Configuration'" } @@ -632,7 +632,8 @@ function Update-PSSignedBuildFolder [Parameter(Mandatory)] [string]$BuildPath, [Parameter(Mandatory)] - [string]$SignedFilesPath + [string]$SignedFilesPath, + [string[]] $RemoveFilter = ('*.pdb', '*.zip') ) # Replace unsigned binaries with signed @@ -643,6 +644,11 @@ function Update-PSSignedBuildFolder Write-Log "replacing $destination with $_" Copy-Item -Path $_ -Destination $destination -force } + + foreach($filter in $RemoveFilter) { + $removePath = Join-Path -Path $BuildPath -ChildPath $filter + Remove-Item -Path $removePath -Recurse -Force + } } @@ -677,7 +683,7 @@ function Expand-PSSignedBuild Restore-PSModuleToBuild -PublishPath $buildPath $psOptionsPath = Join-Path $buildPath -ChildPath 'psoptions.json' - Restore-PSOptions -PSOptionsPath $psOptionsPath -Remove + Restore-PSOptions -PSOptionsPath $psOptionsPath $options = Get-PSOptions @@ -1632,7 +1638,6 @@ function New-ZipPackage throw "Failed to create $zipLocationPath" } } - #TODO: Use .NET Api to do compresss-archive equivalent if the pscmdlet is not present else { Write-Error -Message "Compress-Archive cmdlet is missing in this PowerShell version" @@ -3704,3 +3709,241 @@ function New-GlobalToolNupkg New-NugetPackage -NuSpecPath $_.RootFolder -PackageDestinationPath $DestinationPath } } + +${mainLinuxBuildFolder} = 'pwshLinuxBuild' +${minSizeLinuxBuildFolder} = 'pwshLinuxBuildMinSize' +${arm32LinuxBuildFolder} = 'pwshLinuxBuildArm32' +${arm64LinuxBuildFolder} = 'pwshLinuxBuildArm64' + +<# + Used in Azure DevOps Yaml to package all the linux packages for a channel. +#> +function Invoke-AzDevOpsLinuxPackageCreation { + param( + [switch] + $LTS, + + [Parameter(Mandatory)] + [ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")] + [ValidateNotNullOrEmpty()] + [string]$ReleaseTag, + + [Parameter(Mandatory)] + [ValidateSet('fxdependent', 'alpine', 'deb', 'rpm')] + [String]$BuildType + ) + + if (!${env:SYSTEM_ARTIFACTSDIRECTORY}) { + throw "Must be run in Azure DevOps" + } + + try { + Write-Verbose "Packaging '$BuildType'-LTS:$LTS for $ReleaseTag ..." -Verbose + + Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${mainLinuxBuildFolder}-meta\psoptions.json" + + $releaseTagParam = @{ 'ReleaseTag' = $ReleaseTag } + + switch ($BuildType) { + 'fxdependent' { + Start-PSPackage -Type 'fxdependent' @releaseTagParam -LTS:$LTS + } + 'alpine' { + Start-PSPackage -Type 'tar-alpine' @releaseTagParam -LTS:$LTS + } + 'rpm' { + Start-PSPackage -Type 'rpm' @releaseTagParam -LTS:$LTS + } + default { + Start-PSPackage @releaseTagParam -LTS:$LTS -Type 'deb', 'tar' + } + } + + if ($BuildType -eq 'deb') { + Start-PSPackage -Type tar @releaseTagParam -LTS:$LTS + + Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${minSizeLinuxBuildFolder}-meta\psoptions.json" + + Write-Verbose -Verbose "---- Min-Size ----" + Write-Verbose -Verbose "options.Output: $($options.Output)" + Write-Verbose -Verbose "options.Top $($options.Top)" + + Start-PSPackage -Type min-size @releaseTagParam -LTS:$LTS + + ## Create 'linux-arm' 'tar.gz' package. + ## Note that 'linux-arm' can only be built on Ubuntu environment. + Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${arm32LinuxBuildFolder}-meta\psoptions.json" + Start-PSPackage -Type tar-arm @releaseTagParam -LTS:$LTS + + ## Create 'linux-arm64' 'tar.gz' package. + ## Note that 'linux-arm64' can only be built on Ubuntu environment. + Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${arm64LinuxBuildFolder}-meta\psoptions.json" + Start-PSPackage -Type tar-arm64 @releaseTagParam -LTS:$LTS + } + } + catch { + Get-Error + throw + } +} + +<# + Used in Azure DevOps Yaml to do all the builds needed for all Linux packages for a channel. +#> +function Invoke-AzDevOpsLinuxPackageBuild { + param ( + [Parameter(Mandatory)] + [ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")] + [ValidateNotNullOrEmpty()] + [string]$ReleaseTag, + + [Parameter(Mandatory)] + [ValidateSet('fxdependent', 'alpine', 'deb', 'rpm')] + [String]$BuildType + ) + + if (!${env:SYSTEM_ARTIFACTSDIRECTORY}) { + throw "Must be run in Azure DevOps" + } + + try { + + Write-Verbose "Building '$BuildType' for $ReleaseTag ..." -Verbose + + $releaseTagParam = @{ 'ReleaseTag' = $ReleaseTag } + + $buildParams = @{ Configuration = 'Release'; PSModuleRestore = $true; Restore = $true } + + switch ($BuildType) { + 'fxdependent' { + $buildParams.Add("Runtime", "fxdependent") + } + 'alpine' { + $buildParams.Add("Runtime", 'alpine-x64') + # We are cross compiling, so we can't generate experimental features + $buildParams.Add("SkipExperimentalFeatureGeneration", $true) + } + default { + $buildParams.Add("Crossgen", $true) + } + } + + $buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${mainLinuxBuildFolder}" + Start-PSBuild @buildParams @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json" + # Remove symbol files. + Remove-Item "${buildFolder}\*.pdb" -Force + + if ($BuildType -eq 'deb') { + ## Build 'min-size' + $options = Get-PSOptions + Write-Verbose -Verbose "---- Min-Size ----" + Write-Verbose -Verbose "options.Output: $($options.Output)" + Write-Verbose -Verbose "options.Top $($options.Top)" + $binDir = Join-Path -Path $options.Top -ChildPath 'bin' + if (Test-Path -Path $binDir) { + Write-Verbose -Verbose "Remove $binDir, to get a clean build for min-size package" + Remove-Item -Path $binDir -Recurse -Force + } + + $buildParams['Crossgen'] = $false + $buildParams['ForMinimalSize'] = $true + $buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${minSizeLinuxBuildFolder}" + Start-PSBuild -Clean @buildParams @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json" + # Remove symbol files, xml document files. + Remove-Item "${buildFolder}\*.pdb", "${buildFolder}\*.xml" -Force + + + ## Build 'linux-arm' and create 'tar.gz' package for it. + ## Note that 'linux-arm' can only be built on Ubuntu environment. + $buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${arm32LinuxBuildFolder}" + Start-PSBuild -Configuration Release -Restore -Runtime linux-arm -PSModuleRestore @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json" + # Remove symbol files. + Remove-Item "${buildFolder}\*.pdb" -Force + + $buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${arm64LinuxBuildFolder}" + Start-PSBuild -Configuration Release -Restore -Runtime linux-arm64 -PSModuleRestore @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json" + # Remove symbol files. + Remove-Item "${buildFolder}\*.pdb" -Force + } + } + catch { + Get-Error + throw + } +} + +enum PackageManifestResultStatus { + Mismatch + Match + MissingFromManifest + MissingFromPackage +} + +class PackageManifestResult { + [string] $File + [string] $ExpectedHash + [string] $ActualHash + [PackageManifestResultStatus] $Status +} + +function Test-PackageManifest { + param ( + [Parameter(Mandatory)] + [string] + $PackagePath + ) + + Begin { + $spdxManifestPath = Join-Path $PackagePath -ChildPath "/_manifest/spdx_2.2/manifest.spdx.json" + $man = Get-Content $spdxManifestPath -ErrorAction Stop | convertfrom-json + $inManifest = @() + } + + Process { + Write-Verbose "Processing $($man.files) files..." -verbose + $man.files | ForEach-Object { + $filePath = Join-Path $PackagePath -childPath $_.fileName + $checksumObj = $_.checksums | Where-Object {$_.algorithm -eq 'sha256'} + $sha256 = $checksumObj.checksumValue + $actualHash = $null + $actualHash = (Get-FileHash -Path $filePath -Algorithm sha256 -ErrorAction SilentlyContinue).Hash + $inManifest += $filePath + if($actualHash -ne $sha256) { + $status = [PackageManifestResultStatus]::Mismatch + if (!$actualHash) { + $status = [PackageManifestResultStatus]::MissingFromPackage + } + [PackageManifestResult] $result = @{ + File = $filePath + ExpectedHash = $sha256 + ActualHash = $actualHash + Status = $status + } + Write-Output $result + } + else { + [PackageManifestResult] $result = @{ + File = $filePath + ExpectedHash = $sha256 + ActualHash = $actualHash + Status = [PackageManifestResultStatus]::Match + } + Write-Output $result + } + } + + + Get-ChildItem $PackagePath -recurse | Select-Object -ExpandProperty FullName | foreach-object { + if(!$inManifest -contains $_) { + $actualHash = (get-filehash -Path $_ -algorithm sha256 -erroraction silentlycontinue).Hash + [PackageManifestResult] $result = @{ + File = $_ + ExpectedHash = $null + ActualHash = $actualHash + Status = [PackageManifestResultStatus]::MissingFromManifest + } + Write-Output $result + } + } + } +} diff --git a/tools/packaging/releaseTests/sbom.tests.ps1 b/tools/packaging/releaseTests/sbom.tests.ps1 new file mode 100644 index 00000000000..0e64e6c5042 --- /dev/null +++ b/tools/packaging/releaseTests/sbom.tests.ps1 @@ -0,0 +1,110 @@ +Describe "Verify SBOMs" { + BeforeAll { + Write-Verbose "In Describe BeforeAll" -Verbose + Import-Module $PSScriptRoot/../../../build.psm1 + Import-Module $PSScriptRoot/../packaging.psd1 -Force + $matchCases = @() + $testCases = @() + $missingFromPackageCases = @() + $missingFromManifestCases = @() + Write-Verbose "${env:PACKAGE_FOLDER}" -Verbose + Get-ChildItem $env:PACKAGE_FOLDER -Filter *.zip | ForEach-Object { + Write-Verbose "Found $($_.Name)..." -Verbose + $testCases += @{ + FilePath = $_.FullName + Name = $_.Name + } + } + + foreach($case in $testCases) { + $name = $case.Name + Write-Verbose "Testing $name..." -Verbose + $extractedPath = Join-Path Testdrive:\ -ChildPath ([System.io.path]::GetRandomFileName()) + $null = New-Item -Path $extractedPath -ItemType Directory -Force + Expand-Archive -Path $case.FilePath -DestinationPath $extractedPath + $manifestPath = Join-Path $extractedPath -ChildPath '/_manifest/spdx_2.2/manifest.spdx.json' + It "$name has a BOM" { + $manifestPath | Should -Exist + } + Test-PackageManifest -PackagePath $extractedPath | ForEach-Object { + $status = $_.Status + $expectedHash = $_.ExpectedHash + $actual = $_.ActualHash + $file = $_.File + + switch($status) { + # cover match and mismatch + default { + $matchCases += @{ + Name = $name + File = $file + ActualHash = $actual + ExpectedHash = $ExpectedHash + Status = $status + } + } + "MissingFromPackage" { + $missingFromPackageCases = @{ + Name = $name + File = $file + ActualHash = $actual + ExpectedHash = $ExpectedHash + Status = $status + } + } + "MissingFromManifest" { + $missingFromManifestCases = @{ + Name = $name + File = $file + ActualHash = $actual + ExpectedHash = $ExpectedHash + Status = $status + } + } + } + } + } + } + + Context "Zip files" { + BeforeAll { + Write-Verbose "In Context BeforeAll" -Verbose + } + + It " should have with matching hash" -TestCases $matchCases { + param( + $Name, + $File, + $ActualHash, + $ExpectedHash, + $Status + ) + + $status | Should -Be "Match" -Because "$actualHash should be $expectedHash" + } + + It " should have with matching hash" -TestCases $missingFromPackageCases -Skip:($missingFromPackageCases.Count -eq 0) { + param( + $Name, + $File, + $ActualHash, + $ExpectedHash, + $Status + ) + + $status | Should -Be "Match" -Because "All files in manifest should exist in package" + } + + It "Manifest for should have " -TestCases $missingFromManifestCases -Skip:($missingFromManifestCases.Count -eq 0) { + param( + $Name, + $File, + $ActualHash, + $ExpectedHash, + $Status + ) + + $status | Should -Be "Match" -Because "All files in package should exist in manifest" + } + } +} diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index 73899f117a1..d94f5c9c08a 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -13,6 +13,10 @@ pr: parameters: - name: ForceAzureBlobDelete displayName: Delete Azure Blob + type: string + values: + - true + - false default: false resources: @@ -36,6 +40,10 @@ variables: value: $[counter(variables['branchCounterKey'], 1)] - name: ForceAzureBlobDelete value: ${{ parameters.ForceAzureBlobDelete }} + - name: Github_Build_Repository_Uri + value: https://github.com/powershell/powershell + - name: SBOMGenerator_Formats + value: spdx:2.2 stages: - stage: prep diff --git a/tools/releaseBuild/azureDevOps/releasePipeline.yml b/tools/releaseBuild/azureDevOps/releasePipeline.yml index 8908ccb11b1..39f38ba9a83 100644 --- a/tools/releaseBuild/azureDevOps/releasePipeline.yml +++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml @@ -17,6 +17,8 @@ variables: value : false - name: NugetSecurityAnalysisWarningLevel value: none + - name: skipComponentGovernanceDetection + value: true stages: - stage: ValidateSDK @@ -70,17 +72,26 @@ stages: globalToolExeName: 'pwsh' globalToolPackageName: 'PowerShell.Linux.x64' -- stage: ValidatePkgNames +- stage: StaticPkgValidation dependsOn: [] - displayName: Package Names validation + displayName: Static package validation jobs: - job: ValidatePkgNames + displayName: Validate Package Names pool: vmImage: windows-latest variables: - group: 'Azure Blob variable group' steps: - template: templates/release-ValidatePackageNames.yml + - job: ValidatePkgBOM + displayName: Validate Package BOM + pool: + vmImage: windows-latest + variables: + - group: 'Azure Blob variable group' + steps: + - template: templates/release-ValidatePackageBOM.yml - stage: StartDocker dependsOn: [] @@ -120,7 +131,7 @@ stages: dependsOn: - ValidateSDK - PRCreation - - ValidatePkgNames + - StaticPkgValidation - StartDocker - ManualValidation - ReleaseAutomation diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 30333a23e5c..4265e29c8e2 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -32,36 +32,103 @@ jobs: parameters: ReleaseTagVar: $(ReleaseTagVar) - - powershell: | - Write-Verbose -Verbose "$(build)" + - pwsh: | + # create folder + sudo mkdir /PowerShell - if ('$(build)' -in 'alpine', 'fxdependent' -and '$(ReleaseTagVar)' -match '6.0.*') - { - $vstsCommandString = "vso[task.setvariable variable=SkipBuild]true" - } - else - { - $vstsCommandString = "vso[task.setvariable variable=SkipBuild]false" - } - displayName: 'Skip Alpine or fxdependent for PS v6.0.*' + # make the current user the owner + sudo chown $env:USER /PowerShell + displayName: 'Create /PowerShell' + + - template: cloneToOfficialPath.yml - template: insert-nuget-config-azfeed.yml parameters: - repoRoot: $(REPOROOT) + repoRoot: $(PowerShellRoot) - powershell: | - import-module "$env:REPOROOT/build.psm1" + import-module "$env:POWERSHELLROOT/build.psm1" Sync-PSTags -AddRemoteIfMissing displayName: SyncTags condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) + workingDirectory: $(PowerShellRoot) + + - powershell: | + Import-Module "$env:POWERSHELLROOT/build.psm1" + + Start-PSBootstrap -Package + displayName: 'Bootstrap' + condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) + workingDirectory: $(PowerShellRoot) + + - powershell: | + + Import-Module "$env:POWERSHELLROOT/build.psm1" + Import-Module "$env:POWERSHELLROOT/tools/packaging" + + Invoke-AzDevOpsLinuxPackageBuild -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' + displayName: 'Build' + condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) + workingDirectory: $(PowerShellRoot) + + - template: Sbom.yml@ComplianceRepo + parameters: + BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuild' + Build_Repository_Uri: $(Github_Build_Repository_Uri) + displayName: ${{ parameters.buildName }} SBOM + + - ${{ if eq(variables.build,'deb') }} : + - template: Sbom.yml@ComplianceRepo + parameters: + BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildMinSize' + Build_Repository_Uri: $(Github_Build_Repository_Uri) + displayName: MinSize SBOM + + - ${{ if eq(variables.build,'deb') }} : + - template: Sbom.yml@ComplianceRepo + parameters: + BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm32' + Build_Repository_Uri: $(Github_Build_Repository_Uri) + displayName: Arm32 SBOM + + - ${{ if eq(variables.build,'deb') }} : + - template: Sbom.yml@ComplianceRepo + parameters: + BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm64' + Build_Repository_Uri: $(Github_Build_Repository_Uri) + displayName: Arm64 SBOM + + - powershell: | + Import-Module "$env:POWERSHELLROOT/build.psm1" + Import-Module "$env:POWERSHELLROOT/tools/packaging" + + $metadata = Get-Content "$env:POWERSHELLROOT/tools/metadata.json" -Raw | ConvertFrom-Json + $LTS = $metadata.LTSRelease + Write-Verbose -Verbose -Message "LTS is set to: $LTS" + + Invoke-AzDevOpsLinuxPackageCreation -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' + + if ($LTS) { + Write-Verbose -Verbose "Packaging LTS" + Invoke-AzDevOpsLinuxPackageCreation -LTS -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' + } + displayName: 'Package' + condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) + workingDirectory: $(PowerShellRoot) - powershell: | - $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' - & "$env:REPOROOT/tools/releaseBuild/vstsbuild.ps1" -ReleaseTag $(ReleaseTagVar) -Name '$(build)' - $env:AzDevOpsFeedPAT2 = $null + $linuxPackages = Get-ChildItem "$env:POWERSHELLROOT/powershell*" -Include *.deb,*.rpm,*.tar.gz - displayName: 'Build and package' + $bucket = 'release' + foreach ($linuxPackage in $linuxPackages) + { + $filePath = $linuxPackage.FullName + Write-Verbose "Publishing $filePath to $bucket" -Verbose + Write-Host "##vso[artifact.upload containerfolder=$bucket;artifactname=$bucket]$filePath" + } + displayName: Publish artifacts condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) + workingDirectory: $(PowerShellRoot) - job: upload_${{ parameters.buildName }} displayName: ${{ parameters.uploadDisplayName }} ${{ parameters.buildName }} diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index a276302e466..bfa3a9efdc5 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -81,6 +81,11 @@ jobs: displayName: Merge signed files with Build condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: Sbom.yml@ComplianceRepo + parameters: + BuildDropPath: '$(System.ArtifactsDirectory)/$(SymbolsFolder)' + Build_Repository_Uri: $(Github_Build_Repository_Uri) + - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force diff --git a/tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml b/tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml index 9646cbf6b3b..26229325b82 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml @@ -2,12 +2,12 @@ steps: - pwsh: | $variable = 'releaseTag' $branch = $ENV:BUILD_SOURCEBRANCH - if($branch -notmatch '^.*(release[-/])') + if($branch -notmatch '^.*((release/|rebuild/.*rebuild))') { throw "Branch name is not in release format: '$branch'" } - $releaseTag = $Branch -replace '^.*(release[-/])' + $releaseTag = $Branch -replace '^.*((release|rebuild)/)' $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose Write-Host -Object "##$vstsCommandString" diff --git a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml new file mode 100644 index 00000000000..65eed1ef9e0 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml @@ -0,0 +1,43 @@ +steps: +- checkout: self + clean: true + +- pwsh: | + Get-ChildItem ENV: + displayName: Capture environment + +- template: release-SetReleaseTagAndContainerName.yml + +- pwsh: | + $name = "{0}_{1:x}" -f '$(releaseTag)', (Get-Date).Ticks + Write-Host $name + Write-Host "##vso[build.updatebuildnumber]$name" + displayName: Set Release Name + +- pwsh: | + $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + + & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey) + displayName: Download Azure Artifacts + +- pwsh: | + Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty Name + displayName: Capture Artifact Listing + +- pwsh: | + Install-module Pester -Scope CurrentUser -Force -MaximumVersion 4.99 + displayName: Install Pester + condition: succeededOrFailed() + +- pwsh: | + Import-module './build.psm1' + Import-module './tools/packaging' + $env:PACKAGE_FOLDER = '$(System.ArtifactsDirectory)' + $path = Join-Path -Path $pwd -ChildPath './packageReleaseTests.xml' + $results = invoke-pester -Script './tools/packaging/releaseTests' -OutputFile $path -OutputFormat NUnitXml -PassThru + Write-Host "##vso[results.publish type=NUnit;mergeResults=true;runTitle=Package Release Tests;publishRunAttachments=true;resultFiles=$path;]" + if($results.TotalCount -eq 0 -or $results.FailedCount -gt 0) + { + throw "Package Release Tests failed" + } + displayName: Run packaging release tests diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index fdcde263d05..b804bdc19d9 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -218,6 +218,11 @@ jobs: displayName: Merge ThirdParty signed files with Build condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: Sbom.yml@ComplianceRepo + parameters: + BuildDropPath: '$(System.ArtifactsDirectory)\$(SymbolsFolder)' + Build_Repository_Uri: $(Github_Build_Repository_Uri) + - powershell: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force From 5616c42a83072c58c0f3a374c143e71fa4c0973d Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Fri, 17 Dec 2021 11:28:14 -0800 Subject: [PATCH 07/38] Update SBOM generation (#16641) # Conflicts: # .vsts-ci/misc-analysis/mdSpell.yml # Conflicts: # .vsts-ci/linux.yml # .vsts-ci/mac.yml # .vsts-ci/windows.yml # prquantifier.yaml # tools/cgmanifest.json # tools/findMissingNotices.ps1 # tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml # tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml # tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml # Conflicts: # .vsts-ci/linux.yml --- .vsts-ci/linux.yml | 32 +- .vsts-ci/mac.yml | 21 +- .vsts-ci/misc-analysis/mdSpell.yml | 56 + .vsts-ci/templates/windows-packaging.yml | 1 + .vsts-ci/windows.yml | 17 +- tools/cgmanifest.json | 1344 +++++++++++++++++ tools/findMissingNotices.ps1 | 116 +- .../templates/SetVersionVariables.yml | 22 +- .../azureDevOps/templates/compliance.yml | 2 +- .../templates/compliance/compliance.yml | 90 ++ .../templates/compliance/generateNotice.yml | 83 + .../azureDevOps/templates/json.yml | 2 +- .../azureDevOps/templates/linux.yml | 20 +- .../templates/mac-file-signing.yml | 11 +- .../templates/mac-package-build.yml | 5 +- .../templates/mac-package-signing.yml | 11 +- .../azureDevOps/templates/mac.yml | 2 +- .../azureDevOps/templates/nuget.yml | 2 +- .../templates/windows-hosted-build.yml | 10 +- .../templates/windows-package-signing.yml | 10 +- .../templates/windows-packaging.yml | 5 +- 21 files changed, 1786 insertions(+), 76 deletions(-) create mode 100644 .vsts-ci/misc-analysis/mdSpell.yml create mode 100644 tools/cgmanifest.json create mode 100644 tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml create mode 100644 tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 1ca75f61f9a..c3cec26b846 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -11,9 +11,10 @@ trigger: include: - '*' exclude: - - /.vsts-ci/misc-analysis.yml - - /.github/ISSUE_TEMPLATE/* - - /.dependabot/config.yml + - .vsts-ci/misc-analysis.yml + - .github/ISSUE_TEMPLATE/* + - .dependabot/config.yml + - test/perf/* pr: branches: include: @@ -24,12 +25,17 @@ pr: include: - '*' exclude: + - .dependabot/config.yml + - .github/ISSUE_TEMPLATE/* + - .vsts-ci/misc-analysis.yml + - .vsts-ci/windows.yml + - .vsts-ci/windows/* + - tools/cgmanifest.json + - LICENSE.txt - test/common/markdown/* + - test/perf/* - tools/releaseBuild/* - tools/releaseBuild/azureDevOps/templates/* - - .vsts-ci/misc-analysis.yml - - .github/ISSUE_TEMPLATE/* - - .dependabot/config.yml variables: DOTNET_CLI_TELEMETRY_OPTOUT: 1 @@ -44,14 +50,14 @@ resources: jobs: - template: templates/ci-build.yml parameters: - pool: ubuntu-16.04 + pool: ubuntu-20.04 jobName: linux_build displayName: linux Build - template: templates/nix-test.yml parameters: name: Linux - pool: ubuntu-16.04 + pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: CI parentJobs: @@ -60,7 +66,7 @@ jobs: - template: templates/nix-test.yml parameters: name: Linux - pool: ubuntu-16.04 + pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: CI parentJobs: @@ -69,7 +75,7 @@ jobs: - template: templates/nix-test.yml parameters: name: Linux - pool: ubuntu-16.04 + pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: Others parentJobs: @@ -78,7 +84,7 @@ jobs: - template: templates/nix-test.yml parameters: name: Linux - pool: ubuntu-16.04 + pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: Others parentJobs: @@ -86,12 +92,14 @@ jobs: - template: templates/verify-xunit.yml parameters: - pool: ubuntu-16.04 + pool: ubuntu-20.04 parentJobs: - linux_build - job: CodeCovTestPackage displayName: CodeCoverage and Test Packages + pool: + vmImage: ubuntu-20.04 steps: - powershell: | Import-Module .\tools\ci.psm1 diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index 5448d7be109..298f2b745a3 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -11,10 +11,11 @@ trigger: include: - '*' exclude: - - /tools/releaseBuild/**/* - - /.vsts-ci/misc-analysis.yml - - /.github/ISSUE_TEMPLATE/* - - /.dependabot/config.yml + - tools/releaseBuild/**/* + - .vsts-ci/misc-analysis.yml + - .github/ISSUE_TEMPLATE/* + - .dependabot/config.yml + - test/perf/* pr: branches: include: @@ -25,10 +26,16 @@ pr: include: - '*' exclude: - - test/common/markdown/* - - .vsts-ci/misc-analysis.yml - - .github/ISSUE_TEMPLATE/* - .dependabot/config.yml + - .github/ISSUE_TEMPLATE/* + - .vsts-ci/misc-analysis.yml + - .vsts-ci/windows.yml + - .vsts-ci/windows/* + - tools/cgmanifest.json + - LICENSE.txt + - test/common/markdown/* + - test/perf/* + - tools/packaging/* - tools/releaseBuild/* - tools/releaseBuild/azureDevOps/templates/* diff --git a/.vsts-ci/misc-analysis/mdSpell.yml b/.vsts-ci/misc-analysis/mdSpell.yml new file mode 100644 index 00000000000..e9d046e5e96 --- /dev/null +++ b/.vsts-ci/misc-analysis/mdSpell.yml @@ -0,0 +1,56 @@ +jobs: +- job: markdown + displayName: Markdown Spelling + + pool: + vmImage: ubuntu-20.04 + + variables: + - name: repoPath + value: $(Agent.BuildDirectory)/$(repoFolder) + + steps: + - checkout: self + clean: true + path: $(repoFolder) + + - checkout: ComplianceRepo + + - powershell: | + Get-ChildItem -Path env: + displayName: Capture Environment + condition: succeededOrFailed() + + - bash: | + curl -o- --progress-bar -L https://yarnpkg.com/install.sh | bash + displayName: Bootstrap Yarn + condition: succeededOrFailed() + + - bash: | + sudo yarn global add markdown-spellcheck@0.11.0 + displayName: Install mdspell + condition: succeededOrFailed() + + - bash: | + mdspell '**/*.md' '!**/Pester/**/*.md' '!**/dotnet-tools/**/*.md' --ignore-numbers --ignore-acronyms --report --en-us; + displayName: Test Spelling in Markdown + condition: succeededOrFailed() + workingDirectory: '$(repoPath)' + + - ${{ if not(contains(variables['SYSTEM.COLLECTIONURI'],'mscodehub')) }}: + - pwsh: | + Import-module ./build.psm1 + $path = Join-Path -Path $pwd -ChildPath './commonTestResults.xml' + $results = invoke-pester -Script ./test/common/markdown-lint -OutputFile $path -OutputFormat NUnitXml -PassThru + Write-Host "##vso[results.publish type=NUnit;mergeResults=true;runTitle=Markdown Lint;publishRunAttachments=true;resultFiles=$path;]" + if($results.TotalCount -eq 0 -or $results.FailedCount -gt 0) + { + throw "Markdown tests failed" + } + displayName: Run Markdown Lint Tests + condition: succeededOrFailed() + workingDirectory: '$(repoPath)' + + - template: dailyBuildCompliance.yml@ComplianceRepo + parameters: + sourceScanPath: '$(repoPath)/test/common' diff --git a/.vsts-ci/templates/windows-packaging.yml b/.vsts-ci/templates/windows-packaging.yml index 82219997891..9f96b4884b7 100644 --- a/.vsts-ci/templates/windows-packaging.yml +++ b/.vsts-ci/templates/windows-packaging.yml @@ -61,6 +61,7 @@ jobs: BuildDropPath: '$(System.ArtifactsDirectory)/mainBuild' Build_Repository_Uri: $(build.repository.uri) displayName: SBOM + sourceScanPath: '$(repoPath)\tools' - pwsh: | Import-Module .\tools\ci.psm1 diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index 6554e3f38e5..dc17cbe653e 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -11,9 +11,10 @@ trigger: include: - '*' exclude: - - /.vsts-ci/misc-analysis.yml - - /.github/ISSUE_TEMPLATE/* - - /.dependabot/config.yml + - .vsts-ci/misc-analysis.yml + - .github/ISSUE_TEMPLATE/* + - .dependabot/config.yml + - test/perf/* pr: branches: include: @@ -24,12 +25,16 @@ pr: include: - '*' exclude: - - .vsts-ci/misc-analysis.yml - - .github/ISSUE_TEMPLATE/* - .dependabot/config.yml + - .github/ISSUE_TEMPLATE/* + - .vsts-ci/misc-analysis.yml + - tools/cgmanifest.json + - LICENSE.txt + - test/common/markdown/* + - test/perf/* + - tools/packaging/* - tools/releaseBuild/* - tools/releaseBuild/azureDevOps/templates/* - - test/common/markdown/* variables: GIT_CONFIG_PARAMETERS: "'core.autocrlf=false'" diff --git a/tools/cgmanifest.json b/tools/cgmanifest.json new file mode 100644 index 00000000000..c0db3a5aa94 --- /dev/null +++ b/tools/cgmanifest.json @@ -0,0 +1,1344 @@ +{ + "Registrations": [ + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Markdig.Signed", + "Version": "0.21.1" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.ApplicationInsights", + "Version": "2.15.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.CodeAnalysis.Analyzers", + "Version": "3.0.0" + } + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.CodeAnalysis.Common", + "Version": "3.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.CodeAnalysis.CSharp", + "Version": "3.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.CSharp", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.Management.Infrastructure.Runtime.Unix", + "Version": "2.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.Management.Infrastructure.Runtime.Win", + "Version": "2.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.Management.Infrastructure", + "Version": "2.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.NETCore.Platforms", + "Version": "5.0.4" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.NETCore.Targets", + "Version": "1.1.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.NETCore.Windows.ApiSets", + "Version": "1.0.1" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.PowerShell.Native", + "Version": "7.1.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.Win32.Registry.AccessControl", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.Win32.Registry", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.Win32.SystemEvents", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Microsoft.Windows.Compatibility", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Namotion.Reflection", + "Version": "1.0.14" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "Newtonsoft.Json", + "Version": "12.0.3" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "NJsonSchema", + "Version": "10.2.2" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.Collections", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.Globalization", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.IO", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.Reflection.Extensions", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.Reflection.Primitives", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.Reflection", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.Resources.ResourceManager", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.Runtime.Handles", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.Runtime.InteropServices", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.Runtime", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.Text.Encoding", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.any.System.Threading.Tasks", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.debian.8-x64.runtime.native.System.Security.Cryptography.OpenSsl", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.fedora.23-x64.runtime.native.System.Security.Cryptography.OpenSsl", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.fedora.24-x64.runtime.native.System.Security.Cryptography.OpenSsl", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.linux-arm.runtime.native.System.IO.Ports", + "Version": "5.0.0-rtm.20519.4" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.linux-arm64.runtime.native.System.IO.Ports", + "Version": "5.0.0-rtm.20519.4" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.linux-x64.runtime.native.System.IO.Ports", + "Version": "5.0.0-rtm.20519.4" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.native.System.Data.SqlClient.sni", + "Version": "4.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.native.System.IO.Ports", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.native.System.Security.Cryptography.OpenSsl", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.native.System", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.opensuse.13.2-x64.runtime.native.System.Security.Cryptography.OpenSsl", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.opensuse.42.1-x64.runtime.native.System.Security.Cryptography.OpenSsl", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.osx-x64.runtime.native.System.IO.Ports", + "Version": "5.0.0-rtm.20519.4" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.OpenSsl", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.rhel.7-x64.runtime.native.System.Security.Cryptography.OpenSsl", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.ubuntu.14.04-x64.runtime.native.System.Security.Cryptography.OpenSsl", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.ubuntu.16.04-x64.runtime.native.System.Security.Cryptography.OpenSsl", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.ubuntu.16.10-x64.runtime.native.System.Security.Cryptography.OpenSsl", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.unix.System.Diagnostics.Debug", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.unix.System.Private.Uri", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.unix.System.Runtime.Extensions", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.win-arm64.runtime.native.System.Data.SqlClient.sni", + "Version": "4.4.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.win-x64.runtime.native.System.Data.SqlClient.sni", + "Version": "4.4.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.win-x86.runtime.native.System.Data.SqlClient.sni", + "Version": "4.4.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.win.System.Diagnostics.Debug", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.win.System.Runtime.Extensions", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.win7-x64.Microsoft.NETCore.Windows.ApiSets", + "Version": "1.0.1" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.win7-x86.Microsoft.NETCore.Windows.ApiSets", + "Version": "1.0.1" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "runtime.win7.System.Private.Uri", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.CodeDom", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Collections.Immutable", + "Version": "1.5.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Collections", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.ComponentModel.Composition.Registration", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.ComponentModel.Composition", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Configuration.ConfigurationManager", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Data.DataSetExtensions", + "Version": "4.5.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Data.Odbc", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Data.OleDb", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Data.SqlClient", + "Version": "4.8.3" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Diagnostics.Debug", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Diagnostics.DiagnosticSource", + "Version": "4.6.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Diagnostics.EventLog", + "Version": "5.0.1" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Diagnostics.PerformanceCounter", + "Version": "5.0.1" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.DirectoryServices.AccountManagement", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.DirectoryServices.Protocols", + "Version": "5.0.1" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.DirectoryServices", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Drawing.Common", + "Version": "5.0.2" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Dynamic.Runtime", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Formats.Asn1", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Globalization", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.IO.FileSystem.AccessControl", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.IO.Packaging", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.IO.Pipes.AccessControl", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.IO.Ports", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.IO", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Linq.Expressions", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Linq", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Management", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Memory", + "Version": "4.5.4" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Net.Http.WinHttpHandler", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.ObjectModel", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Private.ServiceModel", + "Version": "4.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Private.Uri", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Reflection.Context", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Reflection.DispatchProxy", + "Version": "4.5.1" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Reflection.Emit.ILGeneration", + "Version": "4.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Reflection.Emit.Lightweight", + "Version": "4.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Reflection.Emit", + "Version": "4.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Reflection.Extensions", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Reflection.Metadata", + "Version": "1.6.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Reflection.Primitives", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Reflection.TypeExtensions", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Reflection", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Resources.ResourceManager", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Runtime.Caching", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Runtime.CompilerServices.Unsafe", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Runtime.Extensions", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Runtime.Handles", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Runtime.InteropServices", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Runtime", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Security.AccessControl", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Security.Cryptography.Cng", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Security.Cryptography.Pkcs", + "Version": "5.0.1" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Security.Cryptography.ProtectedData", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Security.Cryptography.Xml", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Security.Permissions", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Security.Principal.Windows", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.ServiceModel.Duplex", + "Version": "4.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.ServiceModel.Http", + "Version": "4.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.ServiceModel.NetTcp", + "Version": "4.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.ServiceModel.Primitives", + "Version": "4.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.ServiceModel.Security", + "Version": "4.7.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.ServiceModel.Syndication", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.ServiceProcess.ServiceController", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Text.Encoding.CodePages", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Text.Encoding", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Text.Encodings.Web", + "Version": "5.0.1" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Threading.AccessControl", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Threading.Tasks.Extensions", + "Version": "4.5.3" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Threading.Tasks", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Threading", + "Version": "4.3.0" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "nuget", + "Nuget": { + "Name": "System.Windows.Extensions", + "Version": "5.0.0" + } + }, + "DevelopmentDependency": false + } + ] +} diff --git a/tools/findMissingNotices.ps1 b/tools/findMissingNotices.ps1 index f2c8066234a..79960a193d6 100644 --- a/tools/findMissingNotices.ps1 +++ b/tools/findMissingNotices.ps1 @@ -5,14 +5,20 @@ # which is used to generate the notice file. # Requires the module dotnet.project.assets from the PowerShell Gallery authored by @TravisEz13 +param( + [switch] $Fix +) + Import-Module dotnet.project.assets +Import-Module "$PSScriptRoot\..\.github\workflows\GHWorkflowHelper" -Force . "$PSScriptRoot\..\tools\buildCommon\startNativeExecution.ps1" $existingRegistrationTable = @{} -$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable +$cgManifestPath = (Resolve-Path -Path $PSScriptRoot\..\tools\cgmanifest.json).ProviderPath +$existingRegistrationsJson = Get-Content $cgManifestPath | ConvertFrom-Json -AsHashtable $existingRegistrationsJson.Registrations | ForEach-Object { $registration = [Registration]$_ - if($registration.Component) { + if ($registration.Component) { $name = $registration.Component.Name() $existingRegistrationTable.Add($name, $registration) } @@ -70,10 +76,17 @@ Class Nuget { } } +$winDesktopSdk = 'Microsoft.NET.Sdk.WindowsDesktop' +if (!$IsWindows) { + $winDesktopSdk = 'Microsoft.NET.Sdk' + Write-Warning "Always using $winDesktopSdk since this is not windows!!!" +} + function New-NugetComponent { param( [string]$name, - [string]$version + [string]$version, + [switch]$DevelopmentDependency ) $nuget = [Nuget]@{ @@ -87,19 +100,54 @@ function New-NugetComponent { $registration = [Registration]@{ Component = $Component - DevelopmentDependency = $false + DevelopmentDependency = $DevelopmentDependency } return $registration } -$winDesktopSdk = 'Microsoft.NET.Sdk.WindowsDesktop' -if (!$IsWindows) { - $winDesktopSdk = 'Microsoft.NET.Sdk' - Write-Warning "Always using $winDesktopSdk since this is not windows!!!" +$nugetPublicVersionCache = [System.Collections.Generic.Dictionary[string, string]]::new() +function Get-NuGetPublicVersion { + param( + [string]$Name, + [string]$Version + ) + + if($nugetPublicVersionCache.ContainsKey($Name)) { + return $nugetPublicVersionCache[$Name] + } + + try { + [System.Management.Automation.SemanticVersion]$desiredVersion = $Version + } catch { + [Version]$desiredVersion = $Version + } + + $publicVersion = $null + $publicVersion = Find-Package -Name $Name -AllowPrereleaseVersions -source nuget.org -AllVersions -ErrorAction SilentlyContinue | ForEach-Object { + try { + $packageVersion = [System.Management.Automation.SemanticVersion]$_.Version + } catch { + # Fall back to using [version] if it is not a semantic version + $packageVersion = $_.Version + } + + $_ | Add-Member -Name SemVer -MemberType NoteProperty -Value $packageVersion -PassThru + } | Where-Object { $_.SemVer -le $desiredVersion } | Sort-Object -Property semver -Descending | Select-Object -First 1 -ExpandProperty Version + + if(!$publicVersion) { + Write-Warning "No public version found for $Name, using $Version" + $publicVersion = $Version + } + + if(!$nugetPublicVersionCache.ContainsKey($Name)) { + $nugetPublicVersionCache.Add($Name, $publicVersion) + } + + return $publicVersion } -Function Get-CGRegistrations { +function Get-CGRegistrations { param( [Parameter(Mandatory)] [ValidateSet( @@ -120,10 +168,10 @@ Function Get-CGRegistrations { [System.Collections.Generic.Dictionary[string, Registration]] $RegistrationTable ) - $newRegistrations = $Registrations + $registrationChanged = $false - $dotnetTargetName = 'net6.0' - $dotnetTargetNameWin7 = 'net6.0-windows7.0' + $dotnetTargetName = 'net5.0' + $dotnetTargetNameWin7 = 'net5.0-windows7.0' $unixProjectName = 'powershell-unix' $windowsProjectName = 'powershell-win-core' $actualRuntime = $Runtime @@ -169,13 +217,7 @@ Function Get-CGRegistrations { } $null = New-PADrive -Path $PSScriptRoot\..\src\$folder\obj\project.assets.json -Name $folder try { - $targets = Get-ChildItem -Path "${folder}:/targets/$target" -ErrorAction Stop | Where-Object { - $_.Type -eq 'package' -and - $_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and - $_.Name -notlike 'StyleCop*' -and - $_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and - $_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*' - } | select-object -ExpandProperty name + $targets = Get-ChildItem -Path "${folder}:/targets/$target" -ErrorAction Stop | Where-Object { $_.Type -eq 'package' } | select-object -ExpandProperty name } catch { Get-ChildItem -Path "${folder}:/targets" | Out-String | Write-Verbose -Verbose throw @@ -190,23 +232,33 @@ Function Get-CGRegistrations { $parts = ($target -split '\|') $name = $parts[0] $targetVersion = $parts[1] - $pattern = [regex]::Escape($name) + " " - $tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern + $publicVersion = Get-NuGetPublicVersion -Name $name -Version $targetVersion # Add the registration to the cgmanifest if the TPN does not contain the name of the target OR # the exisitng CG contains the registration, because if the existing CG contains the registration, # that might be the only reason it is in the TPN. - if ((!$tpnMatch -or $existingRegistrationTable.ContainsKey($name)) -and !$RegistrationTable.ContainsKey($target)) { - $registration = New-NugetComponent -Name $name -Version $targetVersion + if (!$RegistrationTable.ContainsKey($target)) { + $DevelopmentDependency = $false + if (!$existingRegistrationTable.ContainsKey($name) -or $existingRegistrationTable.$name.Component.Version() -ne $publicVersion) { + $registrationChanged = $true + } + if ($existingRegistrationTable.ContainsKey($name) -and $existingRegistrationTable.$name.DevelopmentDependency) { + $DevelopmentDependency = $true + } + + $registration = New-NugetComponent -Name $name -Version $publicVersion -DevelopmentDependency:$DevelopmentDependency $RegistrationTable.Add($target, $registration) } } + + return $registrationChanged } $registrations = [System.Collections.Generic.Dictionary[string, Registration]]::new() $lastCount = 0 +$registrationChanged = $false foreach ($runtime in "win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win-arm", "linux-arm", "linux-arm64", "osx-arm64", "win-arm64", "win7-x86") { - Get-CGRegistrations -Runtime $runtime -RegistrationTable $registrations + $registrationChanged = (Get-CGRegistrations -Runtime $runtime -RegistrationTable $registrations) -or $registrationChanged $count = $registrations.Count $newCount = $count - $lastCount $lastCount = $count @@ -216,5 +268,19 @@ foreach ($runtime in "win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win-arm" $newRegistrations = $registrations.Keys | Sort-Object | ForEach-Object { $registrations[$_] } $count = $newRegistrations.Count -@{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 | Set-Content $PSScriptRoot\..\cgmanifest.json +$newJson = @{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 +if ($Fix -and $registrationChanged) { + $newJson | Set-Content $cgManifestPath + Set-GWVariable -Name CGMANIFEST_PATH -Value $cgManifestPath +} + +if (!$Fix -and $registrationChanged) { + $temp = Get-GWTempPath + + $tempJson = Join-Path -Path $temp -ChildPath "cgmanifest$((Get-Date).ToString('yyyMMddHHmm')).json" + $newJson | Set-Content $tempJson -Encoding utf8NoBOM + Set-GWVariable -Name CGMANIFEST_PATH -Value $tempJson + throw "cgmanifest is out of date. run ./tools/findMissingNotices.ps1 -Fix. Generated cgmanifest is here: $tempJson" +} + Write-Verbose "$count registrations created!" -Verbose diff --git a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml b/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml index 0b8e5f42417..ec72923231b 100644 --- a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml +++ b/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml @@ -7,21 +7,31 @@ steps: - powershell: | $path = "./build.psm1" + if($env:REPOROOT){ + Write-Verbose "reporoot already set to ${env:REPOROOT}" -Verbose + exit 0 + } + if(Test-Path -Path $path) { - $vstsCommandString = "vso[task.setvariable variable=repoRoot]." - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" + Write-Verbose "reporoot detect at: ." -Verbose + $repoRoot = '.' } else{ $path = "./PowerShell/build.psm1" if(Test-Path -Path $path) { - $vstsCommandString = "vso[task.setvariable variable=repoRoot]./PowerShell" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" + Write-Verbose "reporoot detect at: ./PowerShell" -Verbose + $repoRoot = './PowerShell' } } + if($repoRoot) { + $vstsCommandString = "vso[task.setvariable variable=repoRoot]$repoRoot" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + } else { + Write-Verbose -Verbose "repo not found" + } displayName: 'Set repo Root' - powershell: | diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 280ae3551f6..72df4839913 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -153,5 +153,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(Build.SourcesDirectory)\tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml new file mode 100644 index 00000000000..50a133257dd --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml @@ -0,0 +1,90 @@ +parameters: + - name: parentJobs + type: jobList + +jobs: +- job: compliance + variables: + - name: runCodesignValidationInjection + value : false + - name: NugetSecurityAnalysisWarningLevel + value: none + + # Defines the variables APIScanClient, APIScanTenant and APIScanSecret + - group: PS-PS-APIScan + + displayName: Compliance + dependsOn: + ${{ parameters.parentJobs }} + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 + + # APIScan can take a long time + timeoutInMinutes: 180 + + steps: + - checkout: self + clean: true + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3 + displayName: 'Run CredScan' + inputs: + suppressionsFile: tools/credScan/suppress.json + debugMode: false + continueOnError: true + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 + displayName: 'Run PoliCheck' + inputs: + # targetType F means file or folder and is the only applicable value and the default + targetType: F + # 1 to enable source code comment scanning, which is what we should do for open source + optionsFC: 1 + # recurse + optionsXS: 1 + # run for severity 1, 2, 3 and 4 issues + optionsPE: '1|2|3|4' + # disable history management + optionsHMENABLE: 0 + # Excluclusion access database + optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb' + # Terms Exclusion xml file + optionsUEPath: $(Build.SourcesDirectory)\tools\terms\TermsExclusion.xml + continueOnError: true + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 + displayName: 'Publish Security Analysis Logs to Build Artifacts' + continueOnError: true + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1 + displayName: 'TSA upload to Codebase: PowerShellCore_201906' + inputs: + tsaVersion: TsaV2 + codeBaseName: 'PowerShellCore_201906' + uploadFortifySCA: false + uploadFxCop: false + uploadModernCop: false + uploadPREfast: false + uploadRoslyn: false + uploadTSLint: false + uploadCredScan: true + uploadPoliCheck: true + uploadBinSkim: false + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1 + displayName: 'Create Security Analysis Report' + inputs: + TsvFile: false + APIScan: false + BinSkim: false + CredScan: true + PoliCheck: true + PoliCheckBreakOn: Severity2Above + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(Build.SourcesDirectory)\tools' + snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml b/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml new file mode 100644 index 00000000000..00ffccb1d23 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml @@ -0,0 +1,83 @@ +parameters: + - name: parentJobs + type: jobList + +jobs: +- job: generateNotice + variables: + - name: runCodesignValidationInjection + value : false + - name: NugetSecurityAnalysisWarningLevel + value: none + + displayName: Generate Notice + dependsOn: + ${{ parameters.parentJobs }} + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 + + timeoutInMinutes: 15 + + steps: + - checkout: self + clean: true + + - pwsh: | + [string]$Branch=$env:BUILD_SOURCEBRANCH + $branchOnly = $Branch -replace '^refs/heads/'; + $branchOnly = $branchOnly -replace '[_\-]' + + if ($branchOnly -eq 'master') { + $container = 'tpn' + } else { + $container = "tpn-$branchOnly" + } + + $vstsCommandString = "vso[task.setvariable variable=tpnContainer]$container" + Write-Verbose -Message $vstsCommandString -Verbose + Write-Host -Object "##$vstsCommandString" + displayName: Set ContainerName + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(Build.SourcesDirectory)\tools' + + - task: msospo.ospo-extension.8d7f9abb-6896-461d-9e25-4f74ed65ddb2.notice@0 + displayName: 'NOTICE File Generator' + inputs: + outputfile: '$(System.ArtifactsDirectory)\ThirdPartyNotices.txt' + # output format can be html or text + outputformat: text + # this isn't working + # additionaldata: $(Build.SourcesDirectory)\assets\additionalAttributions.txt + + + - pwsh: | + Get-Content -Raw -Path $(Build.SourcesDirectory)\assets\additionalAttributions.txt | Out-File '$(System.ArtifactsDirectory)\ThirdPartyNotices.txt' -Encoding utf8NoBOM -Force -Append + Get-Content -Raw -Path '$(Build.SourcesDirectory)\assets\additionalAttributions.txt' + displayName: Append Additional Attributions + continueOnError: true + + - pwsh: | + Get-Content -Raw -Path '$(System.ArtifactsDirectory)\ThirdPartyNotices.txt' + displayName: Capture Notice + continueOnError: true + + - task: AzureFileCopy@4 + displayName: 'upload Notice' + inputs: + SourcePath: $(System.ArtifactsDirectory)\ThirdPartyNotices.txt + azureSubscription: '$(AzureFileCopySubscription)' + Destination: AzureBlob + storage: '$(StorageAccount)' + ContainerName: $(tpnContainer) + resourceGroup: '$(StorageResourceGroup)' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(System.ArtifactsDirectory) + artifactName: notice + displayName: Publish notice artifacts diff --git a/tools/releaseBuild/azureDevOps/templates/json.yml b/tools/releaseBuild/azureDevOps/templates/json.yml index 570a658756a..34d329ee7f7 100644 --- a/tools/releaseBuild/azureDevOps/templates/json.yml +++ b/tools/releaseBuild/azureDevOps/templates/json.yml @@ -51,5 +51,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(Build.SourcesDirectory)\tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 4265e29c8e2..7d577386598 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -76,6 +76,9 @@ jobs: BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuild' Build_Repository_Uri: $(Github_Build_Repository_Uri) displayName: ${{ parameters.buildName }} SBOM + PackageName: PowerShell Linux + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)/tools' - ${{ if eq(variables.build,'deb') }} : - template: Sbom.yml@ComplianceRepo @@ -83,6 +86,9 @@ jobs: BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildMinSize' Build_Repository_Uri: $(Github_Build_Repository_Uri) displayName: MinSize SBOM + PackageName: PowerShell Linux Minimum Size + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)/tools' - ${{ if eq(variables.build,'deb') }} : - template: Sbom.yml@ComplianceRepo @@ -90,6 +96,9 @@ jobs: BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm32' Build_Repository_Uri: $(Github_Build_Repository_Uri) displayName: Arm32 SBOM + PackageName: PowerShell Linux Arm32 + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)/tools' - ${{ if eq(variables.build,'deb') }} : - template: Sbom.yml@ComplianceRepo @@ -97,6 +106,9 @@ jobs: BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm64' Build_Repository_Uri: $(Github_Build_Repository_Uri) displayName: Arm64 SBOM + PackageName: PowerShell Linux Arm64 + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)/tools' - powershell: | Import-Module "$env:POWERSHELLROOT/build.psm1" @@ -146,6 +158,8 @@ jobs: value: false - name: NugetSecurityAnalysisWarningLevel value: none + - name: skipComponentGovernanceDetection + value: true steps: - checkout: self @@ -247,9 +261,3 @@ jobs: parameters: artifactPath: '$(Build.StagingDirectory)\signedPackages\release' condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)' - snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index bf158652690..3b0ab36c7f0 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -10,19 +10,28 @@ jobs: name: PowerShell1ES demands: - ImageOverride -equals MMS2019 + variables: - group: ESRP - name: runCodesignValidationInjection value: false - name: NugetSecurityAnalysisWarningLevel value: none + - name: repoFolder + value: PowerShell + - name: repoRoot + value: $(Agent.BuildDirectory)\$(repoFolder) + - name: complianceRepoFolder + value: compliance steps: - checkout: self clean: true + path: $(repoFolder) - checkout: ComplianceRepo clean: true + path: $(complianceRepoFolder) - template: shouldSign.yml @@ -106,5 +115,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(repoRoot)\tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index bfa3a9efdc5..c792ac1c322 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -85,6 +85,9 @@ jobs: parameters: BuildDropPath: '$(System.ArtifactsDirectory)/$(SymbolsFolder)' Build_Repository_Uri: $(Github_Build_Repository_Uri) + PackageName: PowerShell macOS ${{ parameters.buildArchitecture }} + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)/tools' - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force @@ -118,5 +121,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(PowerShellRoot)/tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index 74e28e226b1..7ca73a7cc25 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -16,12 +16,21 @@ jobs: value: false - name: NugetSecurityAnalysisWarningLevel value: none + - name: repoFolder + value: PowerShell + - name: repoRoot + value: $(Agent.BuildDirectory)\$(repoFolder) + - name: complianceRepoFolder + value: compliance + steps: - checkout: self clean: true + path: $(repoFolder) - checkout: ComplianceRepo clean: true + path: $(complianceRepoFolder) - template: shouldSign.yml @@ -118,5 +127,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(repoRoot)/tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index 220b08c4f22..8815c142ddf 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -57,5 +57,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(Build.SourcesDirectory)/tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index cece7e81fdd..fe9a95ee1e8 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -217,4 +217,4 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(PackagePath)' + sourceScanPath: '$(repoRoot)\tools' diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml index 5dac09b03f5..96a86f30394 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -9,8 +9,8 @@ parameters: default: '' jobs: -- job: build_windows_${{ parameters.Architecture }} - displayName: Build Windows - ${{ parameters.Architecture }} +- job: build_windows_${{ parameters.Architecture }}_${{ parameters.BuildConfiguration }} + displayName: Build Windows - ${{ parameters.Architecture }} ${{ parameters.BuildConfiguration }} condition: succeeded() dependsOn: ${{ parameters.parentJob }} pool: @@ -46,7 +46,7 @@ jobs: parameters: repoRoot: $(PowerShellRoot) - - powershell: | + - pwsh: | $runtime = switch ($env:Architecture) { @@ -61,7 +61,7 @@ jobs: tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 -location '$(PowerShellRoot)' -destination '$(Build.ArtifactStagingDirectory)/Symbols_$(Architecture)' -Runtime $runtime -ReleaseTag '$(ReleaseTagVar)' -Symbols displayName: 'Build Windows Universal - $(Architecture) Symbols zip' - - powershell: | + - pwsh: | $packageName = (Get-ChildItem '$(Build.ArtifactStagingDirectory)\Symbols_$(Architecture)').FullName $vstsCommandString = "vso[artifact.upload containerfolder=results;artifactname=results]$packageName" Write-Host ("sending " + $vstsCommandString) @@ -71,5 +71,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(PowerShellRoot)\tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index a2da6215f35..ecb927d86f0 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -15,13 +15,21 @@ jobs: - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE value: 1 - group: ESRP + - name: repoFolder + value: PowerShell + - name: repoRoot + value: $(Agent.BuildDirectory)\$(repoFolder) + - name: complianceRepoFolder + value: compliance steps: - checkout: self clean: true + path: $(repoFolder) - checkout: ComplianceRepo clean: true + path: $(complianceRepoFolder) - template: shouldSign.yml - template: SetVersionVariables.yml @@ -106,5 +114,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(repoRoot)\tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index b804bdc19d9..739a7e5f9a1 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -222,6 +222,9 @@ jobs: parameters: BuildDropPath: '$(System.ArtifactsDirectory)\$(SymbolsFolder)' Build_Repository_Uri: $(Github_Build_Repository_Uri) + PackageName: PowerShell Windows ${{ parameters.Architecture }} ${{ parameters.BuildConfiguration }} + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)\tools' - powershell: | Import-Module $(PowerShellRoot)/build.psm1 -Force @@ -271,7 +274,7 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(PowerShellRoot)\tools' snapshotForceEnabled: true - powershell: | From a1a3fb7efb8e76ad3a0db1444ed5448f4eacd13a Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 18 Dec 2021 12:09:19 -0800 Subject: [PATCH 08/38] Update Images used for release (#16580) # Conflicts: # tools/releaseBuild/azureDevOps/releasePipeline.yml # tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml # tools/releaseBuild/azureDevOps/templates/release-ValidateFxdPackage.yml # tools/releaseBuild/azureDevOps/templates/testartifacts.yml # tools/releaseBuild/azureDevOps/vpackRelease.yml --- .../releaseBuild/azureDevOps/releaseBuild.yml | 2 +- .../azureDevOps/releasePipeline.yml | 38 +++++++--- .../templates/checkAzureContainer.yml | 2 +- .../azureDevOps/templates/compliance.yml | 2 +- .../templates/compliance/compliance.yml | 2 +- .../templates/compliance/generateNotice.yml | 2 +- .../azureDevOps/templates/json.yml | 2 +- .../azureDevOps/templates/linux.yml | 4 +- .../templates/mac-file-signing.yml | 2 +- .../templates/mac-package-signing.yml | 2 +- .../azureDevOps/templates/nuget.yml | 2 +- .../templates/release-GlobalToolTest.yml | 1 + .../templates/release-SDKTests.yml | 1 + .../templates/release-UpdateDepsJson.yml | 7 +- .../azureDevOps/templates/testartifacts.yml | 5 +- .../azureDevOps/templates/vpackReleaseJob.yml | 2 + .../windows-component-governance.yml | 2 +- .../templates/windows-hosted-build.yml | 2 +- .../templates/windows-package-signing.yml | 2 +- .../templates/windows-packaging.yml | 2 +- .../releaseBuild/azureDevOps/vpackRelease.yml | 69 +++++++++++-------- 21 files changed, 95 insertions(+), 58 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index d94f5c9c08a..f44926ecc87 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -177,7 +177,7 @@ stages: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS steps: - checkout: self clean: true diff --git a/tools/releaseBuild/azureDevOps/releasePipeline.yml b/tools/releaseBuild/azureDevOps/releasePipeline.yml index 39f38ba9a83..19f9355a1eb 100644 --- a/tools/releaseBuild/azureDevOps/releasePipeline.yml +++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml @@ -79,6 +79,7 @@ stages: - job: ValidatePkgNames displayName: Validate Package Names pool: + # testing vmImage: windows-latest variables: - group: 'Azure Blob variable group' @@ -87,6 +88,7 @@ stages: - job: ValidatePkgBOM displayName: Validate Package BOM pool: + # testing vmImage: windows-latest variables: - group: 'Azure Blob variable group' @@ -140,8 +142,11 @@ stages: jobs: - deployment: AzureBlobPublic displayName: Make Azure Blob Public - pool: - vmImage: windows-latest + + pool: PowerShell1ES + demands: + - ImageOverride -equals MMS2019TLS + variables: - group: 'Staging_ACR' environment: PSReleaseAzureBlobPublic @@ -153,8 +158,11 @@ stages: - job: GitHubDraft displayName: Create GitHub Draft release - pool: - vmImage: windows-latest + + pool: PowerShell1ES + demands: + - ImageOverride -equals MMS2019TLS + variables: - group: 'Azure Blob variable group' - group: 'AzDevOpsArtifacts' @@ -186,14 +194,20 @@ stages: dependsOn: GitHubManualTasks jobs: - job: PublishNuget - pool: - vmImage: windows-latest + + pool: PowerShell1ES + demands: + - ImageOverride -equals MMS2019TLS + steps: - template: templates/release-ReleaseToNuGet.yml - job: PublishPkgsMsftCom - pool: - vmImage: ubuntu-latest + + pool: PowerShell1ES + demands: + - ImageOverride -equals MMSUbuntu20.04TLS + variables: - group: 'AzDevOpsArtifacts' - group: 'packages.microsoft.com' @@ -262,8 +276,12 @@ stages: displayName: Upload BuildInfoJson jobs: - job: UploadJson - pool: - vmImage: windows-latest + displayName: Upload BuildInfoJson + + pool: PowerShell1ES + demands: + - ImageOverride -equals MMS2019TLS + variables: - group: 'Azure Blob variable group' steps: diff --git a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml index 6cd4ee5cb62..a27e9b8e029 100644 --- a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml +++ b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml @@ -10,7 +10,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS steps: - checkout: self clean: true diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 72df4839913..2831ec7ebf3 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -18,7 +18,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS # APIScan can take a long time timeoutInMinutes: 180 diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml index 50a133257dd..92d5a7daec4 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml @@ -19,7 +19,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS # APIScan can take a long time timeoutInMinutes: 180 diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml b/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml index 00ffccb1d23..d904531d6c7 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml @@ -16,7 +16,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS timeoutInMinutes: 15 diff --git a/tools/releaseBuild/azureDevOps/templates/json.yml b/tools/releaseBuild/azureDevOps/templates/json.yml index 34d329ee7f7..cbea46d9eea 100644 --- a/tools/releaseBuild/azureDevOps/templates/json.yml +++ b/tools/releaseBuild/azureDevOps/templates/json.yml @@ -15,7 +15,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS steps: #- task: @ diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 7d577386598..7dd73772e66 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -10,7 +10,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMSUbuntu20.04 + - ImageOverride -equals MMSUbuntu20.04TLS dependsOn: ${{ parameters.parentJob }} variables: - name: runCodesignValidationInjection @@ -149,7 +149,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS variables: - name: buildName value: ${{ parameters.buildName }} diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index 3b0ab36c7f0..eaee5492050 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -9,7 +9,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS variables: - group: ESRP diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index 7ca73a7cc25..540d20c6069 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -9,7 +9,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS variables: - group: ESRP - name: runCodesignValidationInjection diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index fe9a95ee1e8..444fd437464 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -10,7 +10,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS timeoutInMinutes: 90 diff --git a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml index 3c6eb8b469c..5c389955eca 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml @@ -10,6 +10,7 @@ jobs: - job: ${{ parameters.jobName }} displayName: ${{ parameters.displayName }} pool: + # test vmImage: ${{ parameters.imageName }} steps: - checkout: self diff --git a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml index e7a37a758f3..88b71600b6c 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml @@ -7,6 +7,7 @@ jobs: - job: ${{ parameters.jobName }} displayName: ${{ parameters.displayName }} pool: + # testing vmImage: ${{ parameters.imageName }} steps: - checkout: self diff --git a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml index 4b469e281db..6f20198b63b 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml @@ -1,8 +1,11 @@ jobs: - job: UpdateDepsFiles displayName: Update deps files - pool: - vmImage: windows-latest + + pool: PowerShell1ES + demands: + - ImageOverride -equals MMS2019TLS + variables: - group: 'Azure Blob variable group' steps: diff --git a/tools/releaseBuild/azureDevOps/templates/testartifacts.yml b/tools/releaseBuild/azureDevOps/templates/testartifacts.yml index e8fd061715f..9cef7bd909a 100644 --- a/tools/releaseBuild/azureDevOps/templates/testartifacts.yml +++ b/tools/releaseBuild/azureDevOps/templates/testartifacts.yml @@ -7,12 +7,13 @@ jobs: value: none displayName: Build test artifacts condition: succeeded() - pool: 'Hosted Ubuntu 1604' + pool: + # testing + vmImage: ubuntu-latest steps: - checkout: self clean: true - - pwsh: | Import-Module ./build.psm1 Start-PSBootstrap diff --git a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml index 8cc31ab8df9..1ce45d3d3bb 100644 --- a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml +++ b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml @@ -6,6 +6,8 @@ jobs: displayName: Build and Publish VPack - ${{ parameters.architecture }} condition: succeeded() pool: PowerShell1ES + demands: + - ImageOverride -equals MMS2019TLS steps: - checkout: self clean: true diff --git a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml index b21ab208f1b..5719b4f272c 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml @@ -12,7 +12,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS steps: - checkout: self diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml index 96a86f30394..77e374280c2 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -16,7 +16,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS variables: - name: runCodesignValidationInjection value: false diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index ecb927d86f0..31f1bb0834d 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -10,7 +10,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS variables: - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE value: 1 diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index 739a7e5f9a1..60ac151e073 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -16,7 +16,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals MMS2019TLS variables: - name: BuildConfiguration value: ${{ parameters.BuildConfiguration }} diff --git a/tools/releaseBuild/azureDevOps/vpackRelease.yml b/tools/releaseBuild/azureDevOps/vpackRelease.yml index 70091eac8b5..03482479cbb 100644 --- a/tools/releaseBuild/azureDevOps/vpackRelease.yml +++ b/tools/releaseBuild/azureDevOps/vpackRelease.yml @@ -17,36 +17,47 @@ variables: value: 1 - group: Azure Blob variable group -jobs: -- job: rename - displayName: Name the build - condition: succeeded() - pool: - vmImage: 'windows-latest' - steps: - - checkout: self - clean: true - - - template: ./templates/SetVersionVariables.yml +stages: +- stage: prep + displayName: Create buildInfo and name the Pipeline + jobs: + - job: rename + displayName: Name the build + condition: succeeded() + + pool: PowerShell1ES + demands: + - ImageOverride -equals MMS2019TLS + + steps: + - checkout: self + clean: true + + - template: ./templates/SetVersionVariables.yml + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no + + - powershell: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhss"))" + displayName: Set Build Name for Non-PR + condition: ne(variables['Build.Reason'], 'PullRequest') + +- stage: release + displayName: Release + jobs: + - template: ./templates/vpackReleaseJob.yml parameters: - ReleaseTagVar: $(ReleaseTagVar) + architecture: x64 - - powershell: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhss"))" - displayName: Set Build Name for Non-PR - condition: ne(variables['Build.Reason'], 'PullRequest') - -- template: ./templates/vpackReleaseJob.yml - parameters: - architecture: x64 - -- template: ./templates/vpackReleaseJob.yml - parameters: - architecture: x86 + - template: ./templates/vpackReleaseJob.yml + parameters: + architecture: x86 -- template: ./templates/vpackReleaseJob.yml - parameters: - architecture: arm32 + - template: ./templates/vpackReleaseJob.yml + parameters: + architecture: arm32 -- template: ./templates/vpackReleaseJob.yml - parameters: - architecture: arm64 + - template: ./templates/vpackReleaseJob.yml + parameters: + architecture: arm64 From 8f734634e3e574bbd4bded34f06ca43bea247365 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 20 Dec 2021 11:26:27 -0800 Subject: [PATCH 09/38] Register NuGet source when generating `CGManifest` (#16570) * Register nuget source * update cgmanifest --- tools/findMissingNotices.ps1 | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/findMissingNotices.ps1 b/tools/findMissingNotices.ps1 index 79960a193d6..091acb2ce2a 100644 --- a/tools/findMissingNotices.ps1 +++ b/tools/findMissingNotices.ps1 @@ -13,6 +13,11 @@ Import-Module dotnet.project.assets Import-Module "$PSScriptRoot\..\.github\workflows\GHWorkflowHelper" -Force . "$PSScriptRoot\..\tools\buildCommon\startNativeExecution.ps1" +$packageSourceName = 'findMissingNoticesNugetOrg' +if (!(Get-PackageSource -Name $packageSourceName -ErrorAction SilentlyContinue)) { + $null = Register-PackageSource -Name $packageSourceName -Location https://www.nuget.org/api/v2 -ProviderName NuGet +} + $existingRegistrationTable = @{} $cgManifestPath = (Resolve-Path -Path $PSScriptRoot\..\tools\cgmanifest.json).ProviderPath $existingRegistrationsJson = Get-Content $cgManifestPath | ConvertFrom-Json -AsHashtable @@ -109,7 +114,9 @@ function New-NugetComponent { $nugetPublicVersionCache = [System.Collections.Generic.Dictionary[string, string]]::new() function Get-NuGetPublicVersion { param( + [parameter(Mandatory)] [string]$Name, + [parameter(Mandatory)] [string]$Version ) @@ -124,7 +131,7 @@ function Get-NuGetPublicVersion { } $publicVersion = $null - $publicVersion = Find-Package -Name $Name -AllowPrereleaseVersions -source nuget.org -AllVersions -ErrorAction SilentlyContinue | ForEach-Object { + $publicVersion = Find-Package -Name $Name -AllowPrereleaseVersions -source $packageSourceName -AllVersions -ErrorAction SilentlyContinue | ForEach-Object { try { $packageVersion = [System.Management.Automation.SemanticVersion]$_.Version } catch { From 96bdcfca8cb28e58b3c45eedc94864bed6360ba3 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 26 Jan 2022 11:20:08 -0800 Subject: [PATCH 10/38] Remove all references to `cmake` for the builds in this repo (#16578) # Conflicts: # build.psm1 # test/perf/benchmarks/assets/compiler.test.ps1 # Conflicts: # docs/building/linux.md # Conflicts: # build.psm1 --- build.psm1 | 16 ++++++---------- docs/building/linux.md | 6 ++---- docs/building/macos.md | 2 +- .../microsoft_powershell_alpine3/Dockerfile | 14 -------------- 4 files changed, 9 insertions(+), 29 deletions(-) delete mode 100644 tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile diff --git a/build.psm1 b/build.psm1 index fa87449e6a6..e2f6da1ccad 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1751,11 +1751,7 @@ function Start-PSBootstrap { $Deps = @() if ($Environment.IsUbuntu) { # Build tools - $Deps += "curl", "g++", "cmake", "make" - - if ($BuildLinuxArm) { - $Deps += "gcc-arm-linux-gnueabihf", "g++-arm-linux-gnueabihf" - } + $Deps += "curl", "wget" # .NET Core required runtime libraries $Deps += "libunwind8" @@ -1781,7 +1777,7 @@ function Start-PSBootstrap { } } elseif ($Environment.IsRedHatFamily) { # Build tools - $Deps += "which", "curl", "gcc-c++", "cmake", "make" + $Deps += "which", "curl", "wget" # .NET Core required runtime libraries $Deps += "libicu", "libunwind" @@ -1805,7 +1801,7 @@ function Start-PSBootstrap { } } elseif ($Environment.IsSUSEFamily) { # Build tools - $Deps += "gcc", "cmake", "make" + $Deps += "wget" # Packaging tools if ($Package) { $Deps += "ruby-devel", "rpmbuild", "groff", 'libffi-devel' } @@ -1830,8 +1826,8 @@ function Start-PSBootstrap { $PackageManager = "$sudo port" } - # Build tools - $Deps += "cmake" + # wget for downloading dotnet + $Deps += "wget" # .NET Core required runtime libraries $Deps += "openssl" @@ -1840,7 +1836,7 @@ function Start-PSBootstrap { # ignore exitcode, because they may be already installed Start-NativeExecution ([ScriptBlock]::Create("$PackageManager install $Deps")) -IgnoreExitcode } elseif ($Environment.IsAlpine) { - $Deps += 'libunwind', 'libcurl', 'bash', 'cmake', 'clang', 'build-base', 'git', 'curl' + $Deps += 'libunwind', 'libcurl', 'bash', 'build-base', 'git', 'curl', 'wget' Start-NativeExecution { Invoke-Expression "apk add $Deps" diff --git a/docs/building/linux.md b/docs/building/linux.md index 351f02cef4c..bd402c70fc5 100644 --- a/docs/building/linux.md +++ b/docs/building/linux.md @@ -18,8 +18,7 @@ refer to the [Working with the PowerShell Repository](../git/README.md), ### Toolchain Setup -We use the [.NET Command-Line Interface][dotnet-cli] (`dotnet`) to build the managed components, -and [CMake][] to build the native components. +We use the [.NET Command-Line Interface][dotnet-cli] (`dotnet`) to build the managed components. Installing the toolchain is as easy as running `Start-PSBootstrap` in PowerShell. Of course, this requires a self-hosted copy of PowerShell on Linux. @@ -54,8 +53,7 @@ The `Start-PSBootstrap` function does the following: If you want to use `dotnet` outside of `Start-PSBuild`, add `~/.dotnet` to your `PATH` environment variable. -[dotnet-cli]: https://github.com/dotnet/cli -[CMake]: https://cmake.org/cmake/help/v2.8.12/cmake.html +[dotnet-cli]: https://docs.microsoft.com/dotnet/core/tools/ ## Build using our module diff --git a/docs/building/macos.md b/docs/building/macos.md index d969753e636..987dd465a96 100644 --- a/docs/building/macos.md +++ b/docs/building/macos.md @@ -14,7 +14,7 @@ From `pwsh.exe`, run `Import-Module ./build.psm1` and use `Start-PSBootstrap` to The `Start-PSBootstrap` function does the following: -- Uses `brew` or `port` to install CMake, OpenSSL, and GNU WGet +- Uses `brew` or `port` to install OpenSSL, and GNU WGet - Uninstalls any prior versions of .NET CLI - Downloads and installs .NET Core SDK to `~/.dotnet` diff --git a/tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile b/tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile deleted file mode 100644 index fb1070fcd52..00000000000 --- a/tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -# Docker image file that describes an Centos7 image with PowerShell installed from Microsoft YUM Repo - -FROM mcr.microsoft.com/powershell:alpine-3.12 -LABEL maintainer="PowerShell Team " - -# Install dependencies and clean up -RUN apk update \ - && apk add libunwind libcurl cmake clang build-base git bash curl - -COPY PowerShellPackage.ps1 / - -ENV DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - -ENTRYPOINT [ "pwsh" ] From ef5caa0d4352820a48c407ed246242f700560c02 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 26 Jan 2022 11:47:14 -0800 Subject: [PATCH 11/38] Switch to our custom images for build and release (#16801) # Conflicts: # tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml --- build.psm1 | 2 +- tools/packaging/packaging.psm1 | 14 +++- .../releaseBuild/azureDevOps/releaseBuild.yml | 2 +- .../azureDevOps/releasePipeline.yml | 10 +-- .../templates/checkAzureContainer.yml | 2 +- .../azureDevOps/templates/compliance.yml | 2 +- .../templates/compliance/compliance.yml | 2 +- .../templates/compliance/generateNotice.yml | 2 +- .../azureDevOps/templates/json.yml | 2 +- .../azureDevOps/templates/linux.yml | 4 +- .../templates/mac-file-signing.yml | 2 +- .../templates/mac-package-signing.yml | 2 +- .../azureDevOps/templates/nuget.yml | 2 +- .../templates/release-MsixBundle.yml | 73 +++++++++++++++++++ .../templates/release-UpdateDepsJson.yml | 2 +- .../azureDevOps/templates/vpackReleaseJob.yml | 2 +- .../windows-component-governance.yml | 2 +- .../templates/windows-hosted-build.yml | 2 +- .../templates/windows-package-signing.yml | 2 +- .../templates/windows-packaging.yml | 2 +- .../releaseBuild/azureDevOps/vpackRelease.yml | 2 +- 21 files changed, 108 insertions(+), 27 deletions(-) create mode 100644 tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml diff --git a/build.psm1 b/build.psm1 index e2f6da1ccad..d4cd5b311a0 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1759,7 +1759,7 @@ function Start-PSBootstrap { elseif ($Environment.IsUbuntu18) { $Deps += "libicu60"} # Packaging tools - if ($Package) { $Deps += "ruby-dev", "groff", "libffi-dev" } + if ($Package) { $Deps += "ruby-dev", "groff", "libffi-dev", "rpm" } # Install dependencies # change the fontend from apt-get to noninteractive diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 86809aa8719..2e9560f4df0 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -934,7 +934,15 @@ function New-UnixPackage { try { if ($pscmdlet.ShouldProcess("Create $type package")) { Write-Log "Creating package with fpm..." - $Output = Start-NativeExecution { fpm $Arguments } + try { + $Output = Start-NativeExecution { fpm $Arguments } + } + catch { + Write-Verbose -Message "!!!Handling error in FPM!!!" -Verbose -ErrorAction SilentlyContinue + Write-Verbose -Message "$Output" -Verbose -ErrorAction SilentlyContinue + Get-Error -InputObject $_ + throw + } } } finally { if ($Environment.IsMacOS) { @@ -3782,7 +3790,7 @@ function Invoke-AzDevOpsLinuxPackageCreation { } } catch { - Get-Error + Get-Error -InputObject $_ throw } } @@ -3867,7 +3875,7 @@ function Invoke-AzDevOpsLinuxPackageBuild { } } catch { - Get-Error + Get-Error -InputObject $_ throw } } diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index f44926ecc87..42b1c01d6d9 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -177,7 +177,7 @@ stages: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure steps: - checkout: self clean: true diff --git a/tools/releaseBuild/azureDevOps/releasePipeline.yml b/tools/releaseBuild/azureDevOps/releasePipeline.yml index 19f9355a1eb..55d3837e093 100644 --- a/tools/releaseBuild/azureDevOps/releasePipeline.yml +++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml @@ -145,7 +145,7 @@ stages: pool: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure variables: - group: 'Staging_ACR' @@ -161,7 +161,7 @@ stages: pool: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure variables: - group: 'Azure Blob variable group' @@ -197,7 +197,7 @@ stages: pool: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure steps: - template: templates/release-ReleaseToNuGet.yml @@ -206,7 +206,7 @@ stages: pool: PowerShell1ES demands: - - ImageOverride -equals MMSUbuntu20.04TLS + - ImageOverride -equals PSMMSUbuntu20.04-Secure variables: - group: 'AzDevOpsArtifacts' @@ -280,7 +280,7 @@ stages: pool: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure variables: - group: 'Azure Blob variable group' diff --git a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml index a27e9b8e029..82b88999b29 100644 --- a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml +++ b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml @@ -10,7 +10,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure steps: - checkout: self clean: true diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 2831ec7ebf3..6d72f98c798 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -18,7 +18,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure # APIScan can take a long time timeoutInMinutes: 180 diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml index 92d5a7daec4..e81bdcf3709 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml @@ -19,7 +19,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure # APIScan can take a long time timeoutInMinutes: 180 diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml b/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml index d904531d6c7..0e63dd9007b 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml @@ -16,7 +16,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure timeoutInMinutes: 15 diff --git a/tools/releaseBuild/azureDevOps/templates/json.yml b/tools/releaseBuild/azureDevOps/templates/json.yml index cbea46d9eea..714978022a5 100644 --- a/tools/releaseBuild/azureDevOps/templates/json.yml +++ b/tools/releaseBuild/azureDevOps/templates/json.yml @@ -15,7 +15,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure steps: #- task: @ diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 7dd73772e66..19e0b4ca2dd 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -10,7 +10,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMSUbuntu20.04TLS + - ImageOverride -equals PSMMSUbuntu20.04-Secure dependsOn: ${{ parameters.parentJob }} variables: - name: runCodesignValidationInjection @@ -149,7 +149,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure variables: - name: buildName value: ${{ parameters.buildName }} diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index eaee5492050..8139d9ff819 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -9,7 +9,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure variables: - group: ESRP diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index 540d20c6069..cc7fd8d49ab 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -9,7 +9,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure variables: - group: ESRP - name: runCodesignValidationInjection diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index 444fd437464..e5f4bf4b718 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -10,7 +10,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure timeoutInMinutes: 90 diff --git a/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml b/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml new file mode 100644 index 00000000000..607fda3ad40 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml @@ -0,0 +1,73 @@ +jobs: +- job: CreateMSIXBundle + displayName: Create .msixbundle file + + pool: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-Secure + + variables: + - group: msixTools + - group: 'Azure Blob variable group' + + steps: + - template: release-SetReleaseTagAndContainerName.yml + + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: PowerShellCore + pipeline: '696' + preferTriggeringPipeline: true + runVersion: latestFromBranch + runBranch: '$(Build.SourceBranch)' + artifact: finalResults + patterns: '**/*.msix' + path: '$(Pipeline.Workspace)\releasePipeline\msix' + + - pwsh: | + $cmd = Get-Command makeappx.exe -ErrorAction Ignore + if ($cmd) { + Write-Verbose -Verbose 'makeappx available in PATH' + $exePath = $cmd.Source + } else { + $toolsDir = '$(Pipeline.Workspace)\releasePipeline\tools' + New-Item $toolsDir -Type Directory -Force > $null + Invoke-RestMethod -Uri '$(makeappUrl)' -OutFile "$toolsDir\makeappx.zip" + Expand-Archive "$toolsDir\makeappx.zip" -DestinationPath "$toolsDir\makeappx" -Force + $exePath = "$toolsDir\makeappx\makeappx.exe" + + Write-Verbose -Verbose 'makeappx was installed:' + Get-ChildItem -Path $toolsDir -Recurse + } + + $vstsCommandString = "vso[task.setvariable variable=MakeAppxPath]$exePath" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Install makeappx tool + + - pwsh: | + $sourceDir = '$(Pipeline.Workspace)\releasePipeline\msix' + $file = Get-ChildItem $sourceDir | Select-Object -First 1 + $prefix = ($file.BaseName -split "-win")[0] + $pkgName = "$prefix.msixbundle" + Write-Verbose -Verbose "Creating $pkgName" + + $makeappx = '$(MakeAppxPath)' + $outputDir = "$sourceDir\output" + New-Item $outputDir -Type Directory -Force > $null + & $makeappx bundle /d $sourceDir /p "$outputDir\$pkgName" + + Get-ChildItem -Path $sourceDir -Recurse + $vstsCommandString = "vso[task.setvariable variable=BundleDir]$outputDir" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Create MsixBundle + + - pwsh: | + ## We use AzCopy v8.1 in our release pipeline, see the documentation at: + ## https://docs.microsoft.com/en-us/previous-versions/azure/storage/storage-use-azcopy + + $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + & $azcopy /Source:$(BundleDir) /Dest:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion)-private /DestKey:$(StorageAccountKey) /Pattern:*.msixbundle /Y + displayName: Upload MSIX Bundle package to Az Blob diff --git a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml index 6f20198b63b..bd03ddd247f 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml @@ -4,7 +4,7 @@ jobs: pool: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure variables: - group: 'Azure Blob variable group' diff --git a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml index 1ce45d3d3bb..866f005c374 100644 --- a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml +++ b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml @@ -7,7 +7,7 @@ jobs: condition: succeeded() pool: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure steps: - checkout: self clean: true diff --git a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml index 5719b4f272c..53947655d90 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml @@ -12,7 +12,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure steps: - checkout: self diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml index 77e374280c2..8440d731328 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -16,7 +16,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure variables: - name: runCodesignValidationInjection value: false diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index 31f1bb0834d..e84408c0803 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -10,7 +10,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure variables: - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE value: 1 diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index 60ac151e073..60a720c411e 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -16,7 +16,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure variables: - name: BuildConfiguration value: ${{ parameters.BuildConfiguration }} diff --git a/tools/releaseBuild/azureDevOps/vpackRelease.yml b/tools/releaseBuild/azureDevOps/vpackRelease.yml index 03482479cbb..875e2d25c3c 100644 --- a/tools/releaseBuild/azureDevOps/vpackRelease.yml +++ b/tools/releaseBuild/azureDevOps/vpackRelease.yml @@ -27,7 +27,7 @@ stages: pool: PowerShell1ES demands: - - ImageOverride -equals MMS2019TLS + - ImageOverride -equals PSMMS2019-Secure steps: - checkout: self From abb88cafec20ff002bd9eaf3e5a87e0eca3c8c3f Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 27 Jan 2022 10:45:38 -0800 Subject: [PATCH 12/38] Add Linux package dependencies for packaging (#16807) # Conflicts: # build.psm1 # tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 --- build.psm1 | 59 ++++++++++++++----- .../PowerShellPackage.ps1 | 4 +- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/build.psm1 b/build.psm1 index d4cd5b311a0..d06a303a3b8 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1717,6 +1717,43 @@ function Get-RedHatPackageManager { } } +function Install-GlobalGem { + param( + [Parameter()] + [string] + $Sudo = "", + + [Parameter(Mandatory)] + [string] + $GemName, + + [Parameter(Mandatory)] + [string] + $GemVersion + ) + try { + # We cannot guess if the user wants to run gem install as root on linux and windows, + # but macOs usually requires sudo + $gemsudo = '' + if($environment.IsMacOS -or $env:TF_BUILD) { + $gemsudo = $sudo + } + + Start-NativeExecution ([ScriptBlock]::Create("$gemsudo gem install $GemName -v $GemVersion --no-document")) + + } catch { + Write-Warning "Installation of gem $GemName $GemVersion failed! Must resolve manually." + $logs = Get-ChildItem "/var/lib/gems/*/extensions/x86_64-linux/*/$GemName-*/gem_make.out" | Select-Object -ExpandProperty FullName + foreach ($log in $logs) { + Write-Verbose "Contents of: $log" -Verbose + Get-Content -Raw -Path $log -ErrorAction Ignore | ForEach-Object { Write-Verbose $_ -Verbose } + Write-Verbose "END Contents of: $log" -Verbose + } + + throw + } +} + function Start-PSBootstrap { [CmdletBinding( SupportsShouldProcess=$true, @@ -1759,7 +1796,7 @@ function Start-PSBootstrap { elseif ($Environment.IsUbuntu18) { $Deps += "libicu60"} # Packaging tools - if ($Package) { $Deps += "ruby-dev", "groff", "libffi-dev", "rpm" } + if ($Package) { $Deps += "ruby-dev", "groff", "libffi-dev", "rpm", "g++", "make" } # Install dependencies # change the fontend from apt-get to noninteractive @@ -1783,7 +1820,7 @@ function Start-PSBootstrap { $Deps += "libicu", "libunwind" # Packaging tools - if ($Package) { $Deps += "ruby-devel", "rpm-build", "groff", 'libffi-devel' } + if ($Package) { $Deps += "ruby-devel", "rpm-build", "groff", 'libffi-devel', "gcc-c++" } $PackageManager = Get-RedHatPackageManager @@ -1804,7 +1841,7 @@ function Start-PSBootstrap { $Deps += "wget" # Packaging tools - if ($Package) { $Deps += "ruby-devel", "rpmbuild", "groff", 'libffi-devel' } + if ($Package) { $Deps += "ruby-devel", "rpmbuild", "groff", 'libffi-devel', "gcc" } $PackageManager = "zypper --non-interactive install" $baseCommand = "$sudo $PackageManager" @@ -1845,19 +1882,9 @@ function Start-PSBootstrap { # Install [fpm](https://github.com/jordansissel/fpm) and [ronn](https://github.com/rtomayko/ronn) if ($Package) { - try { - # We cannot guess if the user wants to run gem install as root on linux and windows, - # but macOs usually requires sudo - $gemsudo = '' - if($Environment.IsMacOS -or $env:TF_BUILD) { - $gemsudo = $sudo - } - Start-NativeExecution ([ScriptBlock]::Create("$gemsudo gem install ffi -v 1.12.0 --no-document")) - Start-NativeExecution ([ScriptBlock]::Create("$gemsudo gem install fpm -v 1.11.0 --no-document")) - Start-NativeExecution ([ScriptBlock]::Create("$gemsudo gem install ronn -v 0.7.3 --no-document")) - } catch { - Write-Warning "Installation of fpm and ronn gems failed! Must resolve manually." - } + Install-GlobalGem -Sudo $sudo -GemName "ffi" -GemVersion "1.12.0" + Install-GlobalGem -Sudo $sudo -GemName "fpm" -GemVersion "1.11.0" + Install-GlobalGem -Sudo $sudo -GemName "ronn" -GemVersion "0.7.3" } } diff --git a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 index 41f52956df4..65c5fca279f 100644 --- a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 +++ b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 @@ -73,8 +73,8 @@ try{ Write-Verbose "Sync'ing Tags..." -verbose Sync-PSTags -AddRemoteIfMissing - Write-Verbose "Bootstrapping powershell build..." -verbose - Start-PSBootstrap -Force -Package + Write-Verbose "Bootstrapping powershell build..." -Verbose + Start-PSBootstrap -Force -Package -ErrorAction Stop if ($PSCmdlet.ParameterSetName -eq 'packageSigned') { From 8b83303385225fdde20c5226964ebbe54566262e Mon Sep 17 00:00:00 2001 From: Paul Higinbotham Date: Mon, 31 Jan 2022 12:13:32 -0800 Subject: [PATCH 13/38] Add SBOM manifest to nuget packages (#16711) # Conflicts: # build.psm1 # tools/cgmanifest.json # Conflicts: # build.psm1 # tools/cgmanifest.json # tools/packaging/packaging.psm1 # tools/releaseBuild/azureDevOps/templates/nuget.yml # Conflicts: # tools/packaging/packaging.psm1 # tools/releaseBuild/azureDevOps/templates/nuget.yml --- build.psm1 | 1 + tools/packaging/packaging.psd1 | 2 + tools/packaging/packaging.psm1 | 487 +++++++++++------- .../azureDevOps/templates/nuget-pkg-sbom.yml | 70 +++ .../azureDevOps/templates/nuget.yml | 38 +- 5 files changed, 396 insertions(+), 202 deletions(-) create mode 100644 tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml diff --git a/build.psm1 b/build.psm1 index d06a303a3b8..5fc276ef2ee 100644 --- a/build.psm1 +++ b/build.psm1 @@ -2077,6 +2077,7 @@ function Find-Dotnet() { Push-Location $PSScriptRoot $dotnetCLIInstalledVersion = (dotnet --version) Pop-Location + Write-Verbose -Message "Find-DotNet: dotnetCLIInstalledVersion = $dotnetCLIInstalledVersion" if ($dotnetCLIInstalledVersion -ne $dotnetCLIRequiredVersion) { Write-Warning "The 'dotnet' in the current path can't find SDK version ${dotnetCLIRequiredVersion}, prepending $dotnetPath to PATH." # Globally installed dotnet doesn't have the required SDK version, prepend the user local dotnet location diff --git a/tools/packaging/packaging.psd1 b/tools/packaging/packaging.psd1 index 822bccbc137..52ff6e4cda7 100644 --- a/tools/packaging/packaging.psd1 +++ b/tools/packaging/packaging.psd1 @@ -15,6 +15,8 @@ Copyright="Copyright (c) Microsoft Corporation. All rights reserved." 'New-DotnetSdkContainerFxdPackage' 'New-ExePackage' 'New-GlobalToolNupkg' + 'New-ILNugetPackageSource' + 'New-ILNugetPackageFromSource' 'New-ILNugetPackage' 'New-MSIPatch' 'New-PSBuildZip' diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 2e9560f4df0..28781fc94b1 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1655,6 +1655,9 @@ function New-ZipPackage function CreateNugetPlatformFolder { param( + [Parameter(Mandatory = $true)] + [string] $FileName, + [Parameter(Mandatory = $true)] [string] $Platform, @@ -1666,46 +1669,192 @@ function CreateNugetPlatformFolder ) $destPath = New-Item -ItemType Directory -Path (Join-Path $PackageRuntimesFolder "$Platform/lib/netcoreapp3.1") - $fullPath = Join-Path $PlatformBinPath $file + $fullPath = Join-Path $PlatformBinPath $FileName if (-not(Test-Path $fullPath)) { throw "File not found: $fullPath" } Copy-Item -Path $fullPath -Destination $destPath - Write-Log "Copied $file to $Platform" + Write-Log "Copied $FileName to $Platform at path: $destPath" } <# .SYNOPSIS -Creates NuGet packages containing linux, osx and Windows runtime assemblies. +Creates a CGManifest file containing package dependencies for specified file. + +.PARAMETER FilePath +File path name of CGManifest file to be created. + +.PARAMETER Dependencies +Array list of dependency tuples: +[tuple[ [tuple[string, string]], [tuple[string, string]] ] []] +["Id", "Microsoft.PowerShell.SecretStore"], ["Version", "1.1.1.0"] +#> +function New-CGManifest +{ + param ( + [parameter(Mandatory = $true)] + [string] $FilePath, + + [parameter(Mandatory = $false)] + [tuple[ [tuple[string, string]], [tuple[string, string]] ] []] $Dependencies + ) + + Write-Verbose -Verbose -Message "Creating CGManifest for SBOM: $Filepath" + + $Registrations = @() + + foreach ($dependency in $Dependencies) { + $component = @{ + Component = @{ + Type = "nuget"; + NuGet = @{ + Name = ($dependency.Item1.Item2); Version = ($dependency.Item2.Item2) + } + }; + DevelopmentDependency = "true" + } + + $Registrations += $component + } + + $manifest = @{ Registrations = $Registrations } + $jsonManifest = $manifest | ConvertTo-Json -Depth 10 + + $jsonManifest | Out-File -FilePath $FilePath +} + +function New-FileDependencies +{ + param ( + [parameter(Mandatory = $true)] + [string] $FileBaseName, + + [parameter(Mandatory = $true)] + [string] $PackageVersion + ) + + # Filed a tracking bug for automating generation of dependecy list: https://github.com/PowerShell/PowerShell/issues/6247 + $deps = [System.Collections.ArrayList]::new() + + switch ($FileBaseName) { + 'Microsoft.Management.Infrastructure.CimCmdlets' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + } + + 'Microsoft.PowerShell.Commands.Diagnostics' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + } + + 'Microsoft.PowerShell.Commands.Management' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Security'), [tuple]::Create('version', $PackageVersion))) > $null + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + + 'Microsoft.PowerShell.Commands.Utility' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + + 'Microsoft.PowerShell.ConsoleHost' { + $deps.Add([tuple]::Create( [tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + + 'Microsoft.PowerShell.CoreCLR.Eventing' { + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + + 'Microsoft.PowerShell.SDK' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Management'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Utility'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.ConsoleHost'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Security'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.WSMan.Management'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Diagnostics'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.Management.Infrastructure.CimCmdlets'), [tuple]::Create('version', $PackageVersion))) > $null + } + + 'Microsoft.PowerShell.Security' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + } + + 'Microsoft.WSMan.Management' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.WSMan.Runtime'), [tuple]::Create('version', $PackageVersion))) > $null + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + + 'Microsoft.WSMan.Runtime' { + ## No dependencies + } + + 'System.Management.Automation' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.CoreCLR.Eventing'), [tuple]::Create('version', $PackageVersion))) > $null + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + } + + Write-Output $deps +} + +<# +.SYNOPSIS +Creates nuget package sources for a single provided binary file. .DESCRIPTION -Creates a NuGet package of IL assemblies for unix and windows. -The packages for Microsoft.PowerShell.Commands.Diagnostics, Microsoft.PowerShell.Commands.Management, -Microsoft.PowerShell.Commands.Utility, Microsoft.PowerShell.ConsoleHost, Microsoft.PowerShell.CoreCLR.Eventing, -Microsoft.PowerShell.SDK, Microsoft.PowerShell.Security, Microsoft.WSMan.Management, Microsoft.WSMan.Runtime, -System.Management.Automation are created. +Creates IL assemblies, for a single binary file, to be packaged in a NuGet file. +Includes runtime assemblies for linux and Windows runtime assemblies. + +.PARAMETER FileName +File name of binary to create nuget sources for. .PARAMETER PackagePath -Path where the package will be created. +Path where the package source files will be created. .PARAMETER PackageVersion Version of the created package. .PARAMETER WinFxdBinPath -Path to folder containing Windows framework dependent assemblies. +Path to source folder containing Windows framework dependent assemblies. .PARAMETER LinuxFxdBinPath -Path to folder containing Linux framework dependent assemblies. +Path to source folder containing Linux framework dependent assemblies. .PARAMETER GenAPIToolPath Path to the GenAPI.exe tool. #> -function New-ILNugetPackage +function New-ILNugetPackageSource { [CmdletBinding(SupportsShouldProcess = $true)] - param( + param ( + [Parameter(Mandatory = $true)] + [string] $FileName, [Parameter(Mandatory = $true)] [string] $PackagePath, @@ -1720,12 +1869,20 @@ function New-ILNugetPackage [string] $LinuxFxdBinPath, [Parameter(Mandatory = $true)] - [string] $GenAPIToolPath + [string] $GenAPIToolPath, + + [Parameter(Mandatory = $true)] + [string] $CGManifestPath ) - if (-not $Environment.IsWindows) + if (! $Environment.IsWindows) { - throw "New-ILNugetPackage can be only executed on Windows platform." + throw "New-ILNugetPackageSource can be only executed on Windows platform." + } + + if (! $PSCmdlet.ShouldProcess("Create nuget packages at: $PackagePath")) + { + return } $fileList = @( @@ -1739,8 +1896,7 @@ function New-ILNugetPackage "Microsoft.PowerShell.SDK.dll", "Microsoft.WSMan.Management.dll", "Microsoft.WSMan.Runtime.dll", - "System.Management.Automation.dll", - "Microsoft.PowerShell.MarkdownRender.dll") + "System.Management.Automation.dll") $linuxExceptionList = @( "Microsoft.Management.Infrastructure.CimCmdlets.dll", @@ -1749,196 +1905,164 @@ function New-ILNugetPackage "Microsoft.WSMan.Management.dll", "Microsoft.WSMan.Runtime.dll") - if ($PSCmdlet.ShouldProcess("Create nuget packages at: $PackagePath")) - { - $refBinPath = New-TempFolder - $SnkFilePath = "$RepoRoot\src\signing\visualstudiopublic.snk" + $refBinPath = New-TempFolder + $SnkFilePath = "$RepoRoot\src\signing\visualstudiopublic.snk" - New-ReferenceAssembly -linux64BinPath $LinuxFxdBinPath -RefAssemblyDestinationPath $refBinPath -RefAssemblyVersion $PackageVersion -SnkFilePath $SnkFilePath -GenAPIToolPath $GenAPIToolPath + New-ReferenceAssembly -linux64BinPath $LinuxFxdBinPath -RefAssemblyDestinationPath $refBinPath -RefAssemblyVersion $PackageVersion -SnkFilePath $SnkFilePath -GenAPIToolPath $GenAPIToolPath - foreach ($file in $fileList) - { - $tmpPackageRoot = New-TempFolder - # Remove '.dll' at the end - $fileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($file) - $filePackageFolder = New-Item (Join-Path $tmpPackageRoot $fileBaseName) -ItemType Directory -Force - $packageRuntimesFolder = New-Item (Join-Path $filePackageFolder.FullName 'runtimes') -ItemType Directory + if (! (Test-Path $PackagePath)) { + $null = New-Item -Path $PackagePath -ItemType Directory + } - #region ref - $refFolder = New-Item (Join-Path $filePackageFolder.FullName 'ref/netcoreapp3.1') -ItemType Directory -Force - CopyReferenceAssemblies -assemblyName $fileBaseName -refBinPath $refBinPath -refNugetPath $refFolder -assemblyFileList $fileList - #endregion ref + # Remove '.dll' at the end + $fileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($FileName) + $filePackageFolder = New-Item (Join-Path $PackagePath $fileBaseName) -ItemType Directory -Force + $packageRuntimesFolder = New-Item (Join-Path $filePackageFolder.FullName 'runtimes') -ItemType Directory - $packageRuntimesFolderPath = $packageRuntimesFolder.FullName + Write-Verbose -Verbose "New-ILNugetPackageSource: Creating package source folder for file: $FileName at: $filePackageFolder" + #region ref + $refFolder = New-Item (Join-Path $filePackageFolder.FullName "ref/$script:netCoreRuntime") -ItemType Directory -Force + CopyReferenceAssemblies -assemblyName $fileBaseName -refBinPath $refBinPath -refNugetPath $refFolder -assemblyFileList $fileList + #endregion ref + $packageRuntimesFolderPath = $packageRuntimesFolder.FullName - CreateNugetPlatformFolder -Platform 'win' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $WinFxdBinPath + CreateNugetPlatformFolder -FileName $FileName -Platform 'win' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $WinFxdBinPath - if ($linuxExceptionList -notcontains $file ) - { - CreateNugetPlatformFolder -Platform 'unix' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $LinuxFxdBinPath - } + if ($linuxExceptionList -notcontains $FileName ) + { + CreateNugetPlatformFolder -FileName $FileName -Platform 'unix' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $LinuxFxdBinPath + } - if ($file -eq "Microsoft.PowerShell.SDK.dll") - { - # Copy the '$PSHome\ref' folder to the NuGet package, so 'dotnet publish' can deploy the 'ref' folder to the publish folder. - # This is to make 'Add-Type' work in application that hosts PowerShell. - - $contentFolder = New-Item (Join-Path $filePackageFolder "contentFiles\any\any") -ItemType Directory -Force - $dotnetRefAsmFolder = Join-Path -Path $WinFxdBinPath -ChildPath "ref" - Copy-Item -Path $dotnetRefAsmFolder -Destination $contentFolder -Recurse -Force - Write-Log "Copied the reference assembly folder to contentFiles for the SDK package" - - # Copy the built-in module folders to the NuGet package, so 'dotnet publish' can deploy those modules to the $pshome module path. - # This is for enabling applications that hosts PowerShell to ship the built-in modules. - - $winBuiltInModules = @( - "CimCmdlets", - "Microsoft.PowerShell.Diagnostics", - "Microsoft.PowerShell.Host", - "Microsoft.PowerShell.Management", - "Microsoft.PowerShell.Security", - "Microsoft.PowerShell.Utility", - "Microsoft.WSMan.Management", - "PSDiagnostics" - ) - - $unixBuiltInModules = @( - "Microsoft.PowerShell.Host", - "Microsoft.PowerShell.Management", - "Microsoft.PowerShell.Security", - "Microsoft.PowerShell.Utility" - ) - - $winModuleFolder = New-Item (Join-Path $contentFolder "runtimes\win\lib\netcoreapp3.1\Modules") -ItemType Directory -Force - $unixModuleFolder = New-Item (Join-Path $contentFolder "runtimes\unix\lib\netcoreapp3.1\Modules") -ItemType Directory -Force - - foreach ($module in $winBuiltInModules) { - $source = Join-Path $WinFxdBinPath "Modules\$module" - Copy-Item -Path $source -Destination $winModuleFolder -Recurse -Force - } + if ($FileName -eq "Microsoft.PowerShell.SDK.dll") + { + # Copy the '$PSHOME\ref' folder to the NuGet package, so 'dotnet publish' can deploy the 'ref' folder to the publish folder. + # This is to make 'Add-Type' work in application that hosts PowerShell. - foreach ($module in $unixBuiltInModules) { - $source = Join-Path $LinuxFxdBinPath "Modules\$module" - Copy-Item -Path $source -Destination $unixModuleFolder -Recurse -Force - } + $contentFolder = New-Item (Join-Path $filePackageFolder "contentFiles\any\any") -ItemType Directory -Force + $dotnetRefAsmFolder = Join-Path -Path $WinFxdBinPath -ChildPath "ref" + Copy-Item -Path $dotnetRefAsmFolder -Destination $contentFolder -Recurse -Force + Write-Log "Copied the reference assembly folder to contentFiles for the SDK package" - Write-Log "Copied the built-in modules to contentFiles for the SDK package" - } + # Copy the built-in module folders to the NuGet package, so 'dotnet publish' can deploy those modules to the $pshome module path. + # This is for enabling applications that hosts PowerShell to ship the built-in modules. - #region nuspec - # filed a tracking bug for automating generation of dependecy list: https://github.com/PowerShell/PowerShell/issues/6247 - $deps = [System.Collections.ArrayList]::new() + $winBuiltInModules = @( + "CimCmdlets", + "Microsoft.PowerShell.Diagnostics", + "Microsoft.PowerShell.Host", + "Microsoft.PowerShell.Management", + "Microsoft.PowerShell.Security", + "Microsoft.PowerShell.Utility", + "Microsoft.WSMan.Management", + "PSDiagnostics" + ) - switch ($fileBaseName) { - 'Microsoft.Management.Infrastructure.CimCmdlets' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - } + $unixBuiltInModules = @( + "Microsoft.PowerShell.Host", + "Microsoft.PowerShell.Management", + "Microsoft.PowerShell.Security", + "Microsoft.PowerShell.Utility" + ) - 'Microsoft.PowerShell.Commands.Diagnostics' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - } + $winModuleFolder = New-Item (Join-Path $contentFolder "runtimes\win\lib\$script:netCoreRuntime\Modules") -ItemType Directory -Force + $unixModuleFolder = New-Item (Join-Path $contentFolder "runtimes\unix\lib\$script:netCoreRuntime\Modules") -ItemType Directory -Force - 'Microsoft.PowerShell.Commands.Management' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Security'), [tuple]::Create('version', $PackageVersion))) > $null - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } + foreach ($module in $winBuiltInModules) { + $source = Join-Path $WinFxdBinPath "Modules\$module" + Copy-Item -Path $source -Destination $winModuleFolder -Recurse -Force + } - 'Microsoft.PowerShell.Commands.Utility' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.MarkdownRender'), [tuple]::Create('version', $PackageVersion))) > $null + foreach ($module in $unixBuiltInModules) { + $source = Join-Path $LinuxFxdBinPath "Modules\$module" + Copy-Item -Path $source -Destination $unixModuleFolder -Recurse -Force + } - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } + Write-Log "Copied the built-in modules to contentFiles for the SDK package" + } - 'Microsoft.PowerShell.ConsoleHost' { - $deps.Add([tuple]::Create( [tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } + # Create a CGManifest file that lists all dependencies for this package, which is used when creating the SBOM. + if (! (Test-Path -Path $CGManifestPath)) { + $null = New-Item -Path $CGManifestPath -ItemType Directory + } + $deps = New-FileDependencies -FileBaseName $fileBaseName -PackageVersion $PackageVersion + New-CGManifest -FilePath (Join-Path -Path $CGManifestPath -ChildPath "CGManifest.json") -Dependencies $deps - 'Microsoft.PowerShell.CoreCLR.Eventing' { - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } + if (Test-Path $refBinPath) { + Remove-Item $refBinPath -Recurse -Force -ErrorAction SilentlyContinue + } +} - 'Microsoft.PowerShell.SDK' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Management'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Utility'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.ConsoleHost'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Security'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.WSMan.Management'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Diagnostics'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.Management.Infrastructure.CimCmdlets'), [tuple]::Create('version', $PackageVersion))) > $null - } +<# +.SYNOPSIS +Creates a nuget package file from the provided source path. - 'Microsoft.PowerShell.Security' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - } +.PARAMETER FileName +File name of binary to create nuget package for. - 'Microsoft.WSMan.Management' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.WSMan.Runtime'), [tuple]::Create('version', $PackageVersion))) > $null - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } +.PARAMETER PackagePath +Path for the source files and the created NuGet package file. +#> +function New-ILNugetPackageFromSource +{ + [CmdletBinding(SupportsShouldProcess = $true)] + param ( + [Parameter(Mandatory = $true)] + [string] $FileName, - 'Microsoft.WSMan.Runtime' { - ## No dependencies - } + [Parameter(Mandatory = $true)] + [string] $PackageVersion, - 'System.Management.Automation' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.CoreCLR.Eventing'), [tuple]::Create('version', $PackageVersion))) > $null - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } + [Parameter(Mandatory = $true)] + [string] $PackagePath + ) - 'Microsoft.PowerShell.MarkdownRender' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } - } + if (! $Environment.IsWindows) + { + throw "New-ILNugetPackageFromSource can be only executed on Windows platform." + } - New-NuSpec -PackageId $fileBaseName -PackageVersion $PackageVersion -Dependency $deps -FilePath (Join-Path $filePackageFolder.FullName "$fileBaseName.nuspec") - New-NugetPackage -NuSpecPath $filePackageFolder.FullName -PackageDestinationPath $PackagePath - } + if (! $PSCmdlet.ShouldProcess("Create nuget package for file $FileName at: $PackagePath")) + { + return + } - if (Test-Path $refBinPath) - { - Remove-Item $refBinPath -Recurse -Force -ErrorAction SilentlyContinue - } + $fileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($FileName) - if (Test-Path $tmpPackageRoot) - { - Remove-Item $tmpPackageRoot -Recurse -Force -ErrorAction SilentlyContinue - } + $deps = New-FileDependencies -FileBaseName $fileBaseName -PackageVersion $PackageVersion + + $srcFilePackagePath = Join-Path $PackagePath $fileBaseName + + Write-Verbose -Verbose "New-ILNugetPackageFromSource: Creating nuget package for file: $FileName from source path: $srcFilePackagePath" + + if (! (Test-Path $srcFilePackagePath)) { + $msg = "Expected nuget source path $srcFilePackagePath for file $fileBaseName does not exist." + Write-Verbose -Verbose -Message $msg + throw $msg } + + # Remove the CGManifest file used to create the SBOM. + $cgManifestPath = Join-Path -Path $PackagePath -ChildPath 'CGManifest' + $cgManifestFilePath = Join-Path -Path $cgManifestPath -ChildPath 'CGManifest.json' + if (Test-Path -Path $cgManifestFilePath) + { + Write-Verbose -Verbose "Removing CGManifest file: $cgManifestFilePath" + Remove-Item -Path $cgManifestFilePath -Force -ErrorAction Continue + } + + New-NuSpec -PackageId $fileBaseName -PackageVersion $PackageVersion -Dependency $deps -FilePath (Join-Path $srcFilePackagePath "$fileBaseName.nuspec") + + # Copy icon file to package + Copy-Item -Path $iconPath -Destination "$srcFilePackagePath/$iconFileName" -Verbose + + New-NugetPackage -NuSpecPath $srcFilePackagePath -PackageDestinationPath $PackagePath + + # Remove file nuget package source directory + Remove-Item $srcFilePackagePath -Recurse -Force -ErrorAction SilentlyContinue } <# - Copy the generated reference assemblies to the 'ref/netcoreapp3.0' folder properly. - This is a helper function used by 'New-ILNugetPackage' + Copy the generated reference assemblies to the 'ref/net6.0' folder properly. + This is a helper function used by 'New-ILNugetPackageSource'. #> function CopyReferenceAssemblies { @@ -2040,7 +2164,7 @@ function New-NuSpec { [Parameter(Mandatory = $false)] # An array of tuples of tuples to define the dependencies. # First tuple defines 'id' and value eg: ["id", "System.Data.SqlClient"] - # Second tuple defines 'version' and vale eg: ["version", "4.4.2"] + # Second tuple defines 'version' and value eg: ["version", "4.4.2"] # Both these tuples combined together define one dependency. # An array represents all the dependencies. [tuple[ [tuple[string, string]], [tuple[string, string]] ] []] $Dependency, @@ -2143,6 +2267,9 @@ function New-ReferenceAssembly "Microsoft.PowerShell.ConsoleHost" ) + # Ensure needed dotNet version is available. Find-DotNet does this, and is part of build.psm1 which should already be imported. + Find-DotNet -Verbose + foreach ($assemblyName in $assemblyNames) { Write-Log "Building reference assembly for '$assemblyName'" @@ -2340,7 +2467,7 @@ function GenerateBuildArguments Create a NuGet package from a nuspec. .DESCRIPTION -Creates a NuGet using the nuspec using at the specified folder. +Creates a NuGet using the nuspec at the specified folder. It is expected that the lib / ref / runtime folders are welformed. The genereated NuGet package is copied over to the $PackageDestinationPath diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml new file mode 100644 index 00000000000..fe8c1a872ff --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -0,0 +1,70 @@ +parameters: + - name: PackageVersion + - name: PackagePath + - name: WinFxdPath + - name: LinuxFxdPath + - name: GenAPIToolPath + - name: ListOfFiles + type: object + default: + - Microsoft.Management.Infrastructure.CimCmdlets.dll + - Microsoft.PowerShell.Commands.Diagnostics.dll + - Microsoft.PowerShell.Commands.Management.dll + - Microsoft.PowerShell.Commands.Utility.dll + - Microsoft.PowerShell.ConsoleHost.dll + - Microsoft.PowerShell.CoreCLR.Eventing.dll + - Microsoft.PowerShell.Security.dll + - Microsoft.PowerShell.SDK.dll + - Microsoft.WSMan.Management.dll + - Microsoft.WSMan.Runtime.dll + - System.Management.Automation.dll + +steps: +- ${{ each value in parameters.ListOfFiles }}: + - pwsh: | + $FileName = '${{ value }}' + $FileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($FileName) + $FilePackagePath = Join-Path -Path '${{ parameters.PackagePath }}' -ChildPath $FileBaseName + $CGManifestPath = Join-Path -Path '${{ parameters.PackagePath }}' -ChildPath 'CGManifest' + Write-Verbose -Verbose "FileName to package: $FileName" + Write-Verbose -Verbose "FilePackage path: $FilePackagePath" + Write-Verbose -Verbose "CGManifest path: $CGManifestPath" + # Set SBOM package name + $vstsCommandString = "vso[task.setvariable variable=SbomFilePackageName]${FileBaseName}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + # Set SBOM package path variable + $vstsCommandString = "vso[task.setvariable variable=SbomFilePackagePath]${FilePackagePath}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + # Set CGManifest path variable + $vstsCommandString = "vso[task.setvariable variable=CGManifestPath]${CGManifestPath}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + # Create Nuget package sources + Import-Module -Name $env:REPOROOT\build.psm1 + Import-Module -Name $env:REPOROOT\tools\packaging + Find-DotNet + New-ILNugetPackageSource -File $FileName -PackagePath '${{ parameters.PackagePath }}' -PackageVersion '${{ parameters.PackageVersion }}' -WinFxdBinPath '${{ parameters.WinFxdPath }}' -LinuxFxdBinPath '${{ parameters.LinuxFxdPath }}' -GenAPIToolPath '${{ parameters.GenAPIToolPath }}' -CGManifestPath $CGManifestPath + displayName: 'Create NuGet Package source for single file' + + - template: Sbom.yml@ComplianceRepo + parameters: + BuildDropPath: $(SbomFilePackagePath) + Build_Repository_Uri: 'https://github.com/powershell/powershell' + PackageName: $(SbomFilePackageName) + PackageVersion: ${{ parameters.PackageVersion }} + sourceScanPath: $(CGManifestPath) + displayName: SBOM for NuGetPkg + + - pwsh: | + $FileName = '${{ value }}' + $FileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($FileName) + $FilePackagePath = Join-Path -Path '${{ parameters.PackagePath }}' -ChildPath $FileBaseName + Write-Verbose -Verbose "FileName to package: $FileName" + Write-Verbose -Verbose "FilePackage path: $FilePackagePath" + Import-Module -Name $env:REPOROOT\build.psm1 + Import-Module -Name $env:REPOROOT\tools\packaging + Find-DotNet + New-ILNugetPackageFromSource -FileName $FileName -PackageVersion '${{ parameters.PackageVersion }}' -PackagePath '${{ parameters.PackagePath }}' + displayName: 'Create NuGet Package for single file' diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index e5f4bf4b718..4758498e245 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -107,25 +107,25 @@ jobs: parameters: ReleaseTagVar: $(ReleaseTagVar) - - task: NuGetToolInstaller@0 - displayName: 'Install NuGet 4.9.3' - inputs: - versionSpec: 4.9.3 + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' - - powershell: | - Import-Module $env:REPOROOT\build.psm1 - Import-Module $env:REPOROOT\tools\packaging - Find-Dotnet - New-ILNugetPackage -PackagePath "$(PackagePath)" -PackageVersion "$(Version)" -WinFxdBinPath '$(winFxdPath)' -LinuxFxdBinPath '$(linuxFxdPath)' -GenAPIToolPath "$(GenAPIToolPath)" - displayName: 'Create Nuget Package Folders' + # Create nuget packages along with SBOM manifests. + - template: nuget-pkg-sbom.yml + parameters: + PackageVersion: $(Version) + PackagePath: $(PackagePath) + WinFxdPath: $(winFxdPath) + LinuxFxdPath: $(linuxFxdPath) + GenAPIToolPath: $(GenAPIToolPath) - - powershell: | + - pwsh: | Get-ChildItem $(linuxFxdPath) Get-ChildItem $(winFxdPath) Get-ChildItem $(winFxdWinDesktopPath) displayName: Capture fxd folders - - powershell: | + - pwsh: | Import-Module $env:REPOROOT\build.psm1 Import-Module $env:REPOROOT\tools\packaging Find-Dotnet @@ -136,7 +136,7 @@ jobs: New-GlobalToolNupkg -LinuxBinPath "$(linuxFxdPath)" -WindowsBinPath "$(winFxdPath)" -WindowsDesktopBinPath "$(winFxdWinDesktopPath)" -PackageVersion "$(Version)" -DestinationPath "$(PackagePath)\globaltool" displayName: 'Create Global tool packages' - - powershell: | + - pwsh: | Get-ChildItem "$(PackagePath)" -Recurse displayName: Capture generated packages @@ -149,6 +149,7 @@ jobs: **\*.nupkg useMinimatch: true shouldSign: $(SHOULD_SIGN) + displayName: Sign NuPkg - pwsh: | if (-not (Test-Path '$(System.ArtifactsDirectory)\signed\')) { $null = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)\signed\' } @@ -158,13 +159,6 @@ jobs: condition: eq(variables['SHOULD_SIGN'], 'false') - pwsh: | - if (-not (Test-Path '$(System.ArtifactsDirectory)\signed\')) { $null = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)\signed\' } - Copy-Item -Path '$(PackagePath)\*.nupkg' -Destination '$(System.ArtifactsDirectory)\signed\' -Verbose -Force - Copy-Item -Path '$(PackagePath)\globaltool\*.nupkg' -Destination '$(System.ArtifactsDirectory)\signed\' -Verbose -Force - displayName: Fake copy when not signing - condition: eq(variables['SHOULD_SIGN'], 'false') - - - powershell: | Import-Module "${env:REPOROOT}\build.psm1" -Force Get-ChildItem -Recurse "$(System.ArtifactsDirectory)\signed\*.nupkg" -Verbose | ForEach-Object { Start-NativeExecution -sb { nuget.exe verify -All $_.FullName } } displayName: Verify all packages are signed @@ -183,7 +177,7 @@ jobs: parameters: artifactPath: '$(System.ArtifactsDirectory)\signed' - - powershell: | + - pwsh: | if (-not (Test-Path "$(System.ArtifactsDirectory)\signed\globaltool")) { $null = New-Item -Path "$(System.ArtifactsDirectory)\signed\globaltool" -ItemType Directory -Force @@ -217,4 +211,4 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(repoRoot)\tools' + sourceScanPath: '$(PackagePath)' From 3b6604fe4b962c79ad69d3ae93f7c238d0a79759 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 2 Feb 2022 15:20:41 -0800 Subject: [PATCH 14/38] Remove WiX install (#16834) # Conflicts: # tools/releaseBuild/azureDevOps/templates/windows-packaging.yml --- .../azureDevOps/releasePipeline.yml | 3 +- .../templates/windows-packaging.yml | 64 +++++++------------ 2 files changed, 24 insertions(+), 43 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/releasePipeline.yml b/tools/releaseBuild/azureDevOps/releasePipeline.yml index 55d3837e093..7f761875672 100644 --- a/tools/releaseBuild/azureDevOps/releasePipeline.yml +++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml @@ -143,7 +143,8 @@ stages: - deployment: AzureBlobPublic displayName: Make Azure Blob Public - pool: PowerShell1ES + pool: + name: PowerShell1ES demands: - ImageOverride -equals PSMMS2019-Secure diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index 60a720c411e..404a59c963b 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -9,8 +9,8 @@ parameters: default: '' jobs: -- job: sign_windows_${{ parameters.Architecture }} - displayName: Package Windows - ${{ parameters.Architecture }} +- job: sign_windows_${{ parameters.Architecture }}_${{ parameters.BuildConfiguration }} + displayName: Package Windows - ${{ parameters.Architecture }} ${{ parameters.BuildConfiguration }} condition: succeeded() dependsOn: ${{ parameters.parentJob }} pool: @@ -36,18 +36,14 @@ jobs: - checkout: ComplianceRepo clean: true - - template: shouldSign.yml - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) - - powershell: | - $pkgFilter = if ( '$(Architecture)' -eq 'arm' ) { - "arm32" - } - else { - '$(Architecture)' - } + - template: shouldSign.yml + + - pwsh: | + $pkgFilter = if ( '$(Architecture)' -eq 'arm' ) { "arm32" } else { '$(Architecture)' } $vstsCommandString = "vso[task.setvariable variable=PkgFilter]$pkgFilter" Write-Host ("sending " + $vstsCommandString) @@ -63,27 +59,16 @@ jobs: - template: cloneToOfficialPath.yml - pwsh: | - # cleanup previous install - if((Test-Path "${env:ProgramFiles(x86)}\WiX Toolset xcopy")) { - Remove-Item "${env:ProgramFiles(x86)}\WiX Toolset xcopy" -Recurse -Force + $zipPathString = '$(System.ArtifactsDirectory)\Symbols\results\*$(PkgFilter).zip' + Write-Verbose -Verbose "Zip Path: $zipPathString" + $zipPath = Get-Item $zipPathString + if(@($zipPath).Count -eq 0) { + throw "No files found at '$zipPathString'" + } + elseif(@($zipPath).Count -ne 1) { + $names = $zipPath.Name -join "', '" + throw "multiple files '${names}' found with '${zipPathString}'" } - - $toolsDir = New-Item -ItemType Directory -Path '$(Build.ArtifactStagingDirectory)\tools' - $wixUri = 'https://github.com/wixtoolset/wix3/releases/download/wix311rtm/wix311-binaries.zip' - Invoke-RestMethod -Uri $wixUri -OutFile '$(Build.ArtifactStagingDirectory)\tools\wix.zip' -MaximumRetryCount 5 -RetryIntervalSec 10 - - Import-Module '$(PowerShellRoot)/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/wix.psm1' - Install-WixZip -zipPath '$(Build.ArtifactStagingDirectory)\tools\wix.zip' - - $msixUrl = '$(makeappUrl)' - Invoke-RestMethod -Uri $msixUrl -OutFile '\makeappx.zip' - Expand-Archive '\makeappx.zip' -destination '\' -Force - - displayName: Install packaging tools - - - powershell: | - $zipPath = Get-Item '$(System.ArtifactsDirectory)\Symbols\results\*$(PkgFilter).zip' - Write-Verbose -Verbose "Zip Path: $zipPath" $expandedFolder = $zipPath.BaseName Write-Host "sending.. vso[task.setvariable variable=SymbolsFolder]$expandedFolder" @@ -202,13 +187,14 @@ jobs: **\*.dll useMinimatch: true shouldSign: $(SHOULD_SIGN) + displayName: Sign ThirdParty binaries - - powershell: | + - pwsh: | Get-ChildItem '$(System.ArtifactsDirectory)\thirdPartySigned\*' displayName: Captrue ThirdParty Signed files condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - powershell: | + - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force $signedFilesPath = '$(System.ArtifactsDirectory)\thirdPartySigned' @@ -226,7 +212,7 @@ jobs: PackageVersion: $(Version) sourceScanPath: '$(PowerShellRoot)\tools' - - powershell: | + - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force @@ -245,7 +231,7 @@ jobs: Write-Host "##$vstsCommandString" displayName: Compress signed files - - powershell: | + - pwsh: | $runtime = switch ($env:Architecture) { "x64" { "win7-x64" } @@ -257,13 +243,12 @@ jobs: } $signedPkg = "$(BuildPackagePath)" - Write-Verbose -Verbose -Message "signedPkg = $signedPkg" $(PowerShellRoot)/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 -BuildZip $signedPkg -location '$(PowerShellRoot)' -destination '$(System.ArtifactsDirectory)\pkgSigned' -Runtime $runtime -ReleaseTag '$(ReleaseTagVar)' displayName: 'Build Windows Universal - $(Architecture) Package' - - powershell: | + - pwsh: | Get-ChildItem '$(System.ArtifactsDirectory)\pkgSigned' | ForEach-Object { $packagePath = $_.FullName Write-Host "Uploading $packagePath" @@ -277,17 +262,12 @@ jobs: sourceScanPath: '$(PowerShellRoot)\tools' snapshotForceEnabled: true - - powershell: | + - pwsh: | if ((Test-Path "\PowerShell")) { Remove-Item -Path "\PowerShell" -Force -Recurse -Verbose } else { Write-Verbose -Verbose -Message "No cleanup required." } - - if((Test-Path "${env:ProgramFiles(x86)}\WiX Toolset xcopy")) { - Write-Verbose -Verbose "Cleaning up Wix tools" - Remove-Item "${env:ProgramFiles(x86)}\WiX Toolset xcopy" -Recurse -Force - } displayName: Clean up local Clone condition: always() From e854b87f0023eff1579bf8187f9c83900d298678 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 4 Feb 2022 15:21:42 -0800 Subject: [PATCH 15/38] Ensure alpine and arm SKUs have `powershell.config.json` file with experimental features enabled (#16823) # Conflicts: # build.psm1 --- build.psm1 | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/build.psm1 b/build.psm1 index 5fc276ef2ee..c11a61c4d19 100644 --- a/build.psm1 +++ b/build.psm1 @@ -554,35 +554,33 @@ Fix steps: # publish powershell.config.json $config = @{} - if ($environment.IsWindows) { + + if ($Options.Runtime -like "*win*") { + # Execution Policy is only supported on Windows $config = @{ "Microsoft.PowerShell:ExecutionPolicy" = "RemoteSigned"; - "WindowsPowerShellCompatibilityModuleDenyList" = @("PSScheduledJob","BestPractices","UpdateServices") } + "WindowsPowerShellCompatibilityModuleDenyList" = @("PSScheduledJob", "BestPractices", "UpdateServices") + } } - # When building preview, we want the configuration to enable all experiemental features by default - # ARM is cross compiled, so we can't run pwsh to enumerate Experimental Features if (-not $SkipExperimentalFeatureGeneration -and (Test-IsPreview $psVersion) -and - -not (Test-IsReleaseCandidate $psVersion) -and - -not $Runtime.Contains("arm") -and - -not ($Runtime -like 'fxdependent*')) { - - $json = & $publishPath\pwsh -noprofile -command { - $expFeatures = [System.Collections.Generic.List[string]]::new() - Get-ExperimentalFeature | ForEach-Object { $expFeatures.Add($_.Name) } - - # Make sure ExperimentalFeatures from modules in PSHome are added - # https://github.com/PowerShell/PowerShell/issues/10550 - @("PSDesiredStateConfiguration.InvokeDscResource") | ForEach-Object { - if (!$expFeatures.Contains($_)) { - $expFeatures.Add($_) - } - } + -not (Test-IsReleaseCandidate $psVersion) + ) { - ConvertTo-Json $expFeatures.ToArray() + $ExperimentalFeatureJsonFilePath = if ($Options.Runtime -like "*win*") { + "$PSScriptRoot/experimental-feature-windows.json" + } else { + "$PSScriptRoot/experimental-feature-linux.json" + } + + if (-not (Test-Path $ExperimentalFeatureJsonFilePath)) { + throw "ExperimentalFeatureJsonFilePath: $ExperimentalFeatureJsonFilePath does not exist" } + $json = Get-Content -Raw $ExperimentalFeatureJsonFilePath $config += @{ ExperimentalFeatures = ([string[]] ($json | ConvertFrom-Json)) } + } else { + Write-Warning -Message "Experimental features are not enabled in powershell.config.json file" } if ($config.Count -gt 0) { From a049aecc47aaccca02b027d453505d9f72c9d92b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 3 Feb 2022 17:25:49 -0800 Subject: [PATCH 16/38] Update experimental feature json files (#16838) --- experimental-feature-linux.json | 14 ++++++++++++++ experimental-feature-windows.json | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 experimental-feature-linux.json create mode 100644 experimental-feature-windows.json diff --git a/experimental-feature-linux.json b/experimental-feature-linux.json new file mode 100644 index 00000000000..12df4c5db5c --- /dev/null +++ b/experimental-feature-linux.json @@ -0,0 +1,14 @@ +[ + "PSAMSIMethodInvocationLogging", + "PSAnsiRenderingFileInfo", + "PSCleanBlock", + "PSCommandNotFoundSuggestion", + "PSExec", + "PSLoadAssemblyFromNativeCode", + "PSNativeCommandArgumentPassing", + "PSNativeCommandErrorActionPreference", + "PSNativePSPathResolution", + "PSRemotingSSHTransportErrorHandling", + "PSStrictModeAssignment", + "PSSubsystemPluginModel" +] diff --git a/experimental-feature-windows.json b/experimental-feature-windows.json new file mode 100644 index 00000000000..12df4c5db5c --- /dev/null +++ b/experimental-feature-windows.json @@ -0,0 +1,14 @@ +[ + "PSAMSIMethodInvocationLogging", + "PSAnsiRenderingFileInfo", + "PSCleanBlock", + "PSCommandNotFoundSuggestion", + "PSExec", + "PSLoadAssemblyFromNativeCode", + "PSNativeCommandArgumentPassing", + "PSNativeCommandErrorActionPreference", + "PSNativePSPathResolution", + "PSRemotingSSHTransportErrorHandling", + "PSStrictModeAssignment", + "PSSubsystemPluginModel" +] From 3dad9602f3bf4c22cb7e28b5bf791f2f042872fd Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 23 Feb 2022 15:03:28 -0800 Subject: [PATCH 17/38] Opt-in to build security monitoring (#16911) value: true Opt in to security monitoring --- tools/releaseBuild/azureDevOps/releaseBuild.yml | 2 ++ tools/releaseBuild/azureDevOps/releasePipeline.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index 42b1c01d6d9..c6a759d1395 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -44,6 +44,8 @@ variables: value: https://github.com/powershell/powershell - name: SBOMGenerator_Formats value: spdx:2.2 + - name: BUILDSECMON_OPT_IN + value: true stages: - stage: prep diff --git a/tools/releaseBuild/azureDevOps/releasePipeline.yml b/tools/releaseBuild/azureDevOps/releasePipeline.yml index 7f761875672..580495054a2 100644 --- a/tools/releaseBuild/azureDevOps/releasePipeline.yml +++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml @@ -19,6 +19,8 @@ variables: value: none - name: skipComponentGovernanceDetection value: true + - name: BUILDSECMON_OPT_IN + value: true stages: - stage: ValidateSDK From 790dd8e23d9401fb4e4169194a667277ac66d2b1 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Thu, 24 Feb 2022 15:57:36 -0800 Subject: [PATCH 18/38] Infrastructure changes from v7.3.0-preview.2 # Conflicts: # tools/releaseBuild/azureDevOps/templates/nuget.yml --- build.psm1 | 26 +++++++++++++++++++ tools/packaging/packaging.psm1 | 2 -- .../azureDevOps/releasePipeline.yml | 12 ++++++--- .../azureDevOps/templates/nuget.yml | 11 ++++---- .../templates/release-CreateGitHubDraft.yml | 11 +++++++- .../templates/release-GlobalToolTest.yml | 20 ++++++++++++-- .../templates/release-MsixBundle.yml | 11 ++++++-- .../templates/release-SDKTests.yml | 2 +- .../templates/release-UpdateDepsJson.yml | 3 ++- .../templates/release-ValidatePackageBOM.yml | 8 +++++- .../release-ValidatePackageNames.yml | 8 +++++- .../azureDevOps/templates/vpackReleaseJob.yml | 11 ++++++-- 12 files changed, 103 insertions(+), 22 deletions(-) diff --git a/build.psm1 b/build.psm1 index c11a61c4d19..27bdbd29543 100644 --- a/build.psm1 +++ b/build.psm1 @@ -3217,3 +3217,29 @@ function New-NugetConfigFile Set-Content -Path (Join-Path $Destination 'nuget.config') -Value $content -Force } + +function Install-AzCopy { + $testPath = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + if (Test-Path $testPath) { + Write-Verbose "AzCopy already installed" -Verbose + return + } + + $destination = "$env:TEMP\azcopy81.msi" + Invoke-WebRequest "https://aka.ms/downloadazcopy" -OutFile $destination + Start-Process -FilePath $destination -ArgumentList "/quiet" -Wait +} + +function Find-AzCopy { + $searchPaths = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + + foreach ($filter in $searchPaths) { + $azCopy = Get-ChildItem -Path $filter -Recurse -ErrorAction SilentlyContinue | Select-Object -ExpandProperty FullName -First 1 + if ($azCopy) { + return $azCopy + } + } + + $azCopy = Get-Command -Name azCopy -ErrorAction Stop | Select-Object -First 1 + return $azCopy.Path +} diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 28781fc94b1..0f4e63ceb03 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -3955,8 +3955,6 @@ function Invoke-AzDevOpsLinuxPackageBuild { } 'alpine' { $buildParams.Add("Runtime", 'alpine-x64') - # We are cross compiling, so we can't generate experimental features - $buildParams.Add("SkipExperimentalFeatureGeneration", $true) } default { $buildParams.Add("Crossgen", $true) diff --git a/tools/releaseBuild/azureDevOps/releasePipeline.yml b/tools/releaseBuild/azureDevOps/releasePipeline.yml index 580495054a2..4e941483315 100644 --- a/tools/releaseBuild/azureDevOps/releasePipeline.yml +++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml @@ -162,7 +162,8 @@ stages: - job: GitHubDraft displayName: Create GitHub Draft release - pool: PowerShell1ES + pool: + name: PowerShell1ES demands: - ImageOverride -equals PSMMS2019-Secure @@ -198,7 +199,8 @@ stages: jobs: - job: PublishNuget - pool: PowerShell1ES + pool: + name: PowerShell1ES demands: - ImageOverride -equals PSMMS2019-Secure @@ -207,7 +209,8 @@ stages: - job: PublishPkgsMsftCom - pool: PowerShell1ES + pool: + name: PowerShell1ES demands: - ImageOverride -equals PSMMSUbuntu20.04-Secure @@ -281,7 +284,8 @@ stages: - job: UploadJson displayName: Upload BuildInfoJson - pool: PowerShell1ES + pool: + name: PowerShell1ES demands: - ImageOverride -equals PSMMS2019-Secure diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index 4758498e245..96ce980919a 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -51,13 +51,14 @@ jobs: Write-Host "##$vstsCommandString" displayName: 'Find SDK version from global.json' - - task: DotNetCoreInstaller@0 - displayName: 'Use .NET Core SDK from global.json' - inputs: - version: '$(SDKVersion)' + - pwsh: | + Import-Module "$env:REPOROOT/build.psm1" -Force + # We just need .NET but we fixed this in an urgent situation. + Start-PSBootStrap -Verbose + displayName: Bootstrap - task: DownloadBuildArtifacts@0 - displayName: 'Download PowerShell build artifacts' + displayName: 'Download PowerShell build artifacts - finalResults' inputs: buildType: current downloadType: single diff --git a/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml b/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml index 3b316e84cbf..9f614e90c79 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml @@ -1,10 +1,19 @@ steps: +- checkout: self + clean: true + - download: none - template: release-SetReleaseTagAndContainerName.yml - pwsh: | - $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + Install-AzCopy + displayName: Install AzCopy + +- pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + $azcopy = Find-AzCopy & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey) displayName: Download Azure Artifacts diff --git a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml index 5c389955eca..89725b5fca8 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml @@ -47,7 +47,7 @@ jobs: - pwsh: | $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - Start-PSBootstrap -Force + Start-PSBootstrap $toolPath = New-Item -ItemType Directory "$(System.DefaultWorkingDirectory)/toolPath" | Select-Object -ExpandProperty FullName @@ -72,12 +72,28 @@ jobs: - pwsh: | Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - Start-PSBootstrap -Force + Start-PSBootstrap $exeName = if ($IsWindows) { "pwsh.exe" } else { "pwsh" } $toolPath = "$(System.DefaultWorkingDirectory)/toolPath/${{ parameters.globalToolExeName }}" + $source = (get-command -Type Application -Name dotnet | Select-Object -First 1 -ExpandProperty source) + $target = (Get-ChildItem $source).target + + # If we find a symbolic link for dotnet, then we need to split the filename off the target. + if ($target) { + Write-Verbose -Verbose "Splitting target: $target" + $target = Split-Path $target + } + + Write-Verbose -Verbose "target is set as $target" + + $env:DOTNET_ROOT = (resolve-path -Path (Join-Path (split-path $source) $target)).ProviderPath + + Write-Verbose -Verbose "DOTNET_ROOT: $env:DOTNET_ROOT" + Get-ChildItem $env:DOTNET_ROOT + $versionFound = & $toolPath -c '$PSVersionTable.PSVersion.ToString()' if ( '$(PowerShellVersion)' -ne $versionFound) diff --git a/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml b/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml index 607fda3ad40..c883bc0ff0b 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml @@ -2,7 +2,8 @@ jobs: - job: CreateMSIXBundle displayName: Create .msixbundle file - pool: PowerShell1ES + pool: + name: PowerShell1ES demands: - ImageOverride -equals PSMMS2019-Secure @@ -64,10 +65,16 @@ jobs: Write-Host "##$vstsCommandString" displayName: Create MsixBundle + - pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + Install-AzCopy + displayName: Install AzCopy + - pwsh: | ## We use AzCopy v8.1 in our release pipeline, see the documentation at: ## https://docs.microsoft.com/en-us/previous-versions/azure/storage/storage-use-azcopy - $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + $azcopy = Find-AzCopy & $azcopy /Source:$(BundleDir) /Dest:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion)-private /DestKey:$(StorageAccountKey) /Pattern:*.msixbundle /Y displayName: Upload MSIX Bundle package to Az Blob diff --git a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml index 88b71600b6c..a190f59c05e 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml @@ -47,7 +47,7 @@ jobs: - pwsh: | $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - Start-PSBootstrap -Force + Start-PSBootstrap $localLocation = "$(Pipeline.Workspace)/releasePipeline/finalResults" $xmlElement = @" diff --git a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml index bd03ddd247f..8f52bcb358f 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml @@ -2,7 +2,8 @@ jobs: - job: UpdateDepsFiles displayName: Update deps files - pool: PowerShell1ES + pool: + name: PowerShell1ES demands: - ImageOverride -equals PSMMS2019-Secure diff --git a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml index 65eed1ef9e0..4778ebcaa1c 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml @@ -15,7 +15,13 @@ steps: displayName: Set Release Name - pwsh: | - $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + Install-AzCopy + displayName: Install AzCopy + +- pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + $azcopy = Find-AzCopy & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey) displayName: Download Azure Artifacts diff --git a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml index d0276ab11fe..2b1d3bb2e57 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml @@ -12,7 +12,13 @@ steps: displayName: Set Release Name - pwsh: | - $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + Install-AzCopy + displayName: Install AzCopy + +- pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + $azcopy = Find-AzCopy & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey) displayName: Download Azure Artifacts diff --git a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml index 866f005c374..54782d302bb 100644 --- a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml +++ b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml @@ -5,7 +5,8 @@ jobs: - job: vpack_${{ parameters.architecture }} displayName: Build and Publish VPack - ${{ parameters.architecture }} condition: succeeded() - pool: PowerShell1ES + pool: + name: PowerShell1ES demands: - ImageOverride -equals PSMMS2019-Secure steps: @@ -17,7 +18,13 @@ jobs: ReleaseTagVar: $(ReleaseTagVar) - pwsh: | - $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + Install-AzCopy + displayName: Install AzCopy + + - pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + $azcopy = Find-AzCopy Write-Host "running: $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:****** /Pattern:PowerShell-$(Version)-win-${{ parameters.architecture }}.zip /Z:$(AGENT.TEMPDIRECTORY)" & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey) /Pattern:PowerShell-$(Version)-win-${{ parameters.architecture }}.zip /Z:$(AGENT.TEMPDIRECTORY) From c35b8e641d72ab464462f845a622fb5863c949ae Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 18 Oct 2021 13:10:59 -0700 Subject: [PATCH 19/38] Only upload stable `buildinfo` for stable releases (#16251) # Conflicts: # tools/metadata.json # Conflicts: # tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml # Conflicts: # tools/metadata.json # tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml --- tools/metadata.json | 3 ++- .../templates/release-BuildJson.yml | 25 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/tools/metadata.json b/tools/metadata.json index ccf364efbdf..30491b49f76 100644 --- a/tools/metadata.json +++ b/tools/metadata.json @@ -4,5 +4,6 @@ "ServicingReleaseTag": "v6.1.6", "ReleaseTag": "v6.2.4", "NextReleaseTag": "v7.0.0-preview.7", - "LTSRelease": true + "LTSRelease": true, + "StableRelease": false } diff --git a/tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml b/tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml index f03e8f25d7c..966def754f8 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-BuildJson.yml @@ -21,12 +21,19 @@ steps: $dateTime = [datetime]::UtcNow $dateTime = [datetime]::new($dateTime.Ticks - ($dateTime.Ticks % [timespan]::TicksPerSecond), $dateTime.Kind) + $metadata = Get-Content ./tools/metadata.json | ConvertFrom-Json + $stableRelease = $metadata.StableRelease + $ltsRelease = $metadata.LTSRelease $buildInfo = Get-Content $jsonFile | ConvertFrom-Json $buildInfo.ReleaseDate = $dateTime $targetFile = "$ENV:PIPELINE_WORKSPACE/$fileName" ConvertTo-Json -InputObject $buildInfo | Out-File $targetFile -Encoding ascii + if ($stableRelease) { + $vstsCommandCopyLTSBuildInfo = "vso[task.setvariable variable=CopyStableBuildInfo]YES" + } + $vstsCommandString = "vso[task.setvariable variable=BuildInfoJsonFile]$targetFile" Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" @@ -36,19 +43,12 @@ steps: $vstsCommandCopyLTSBuildInfo = "vso[task.setvariable variable=CopyLTSBuildInfo]NO" - if ($fileName -eq "stable.json") + if ($fileName -eq "stable.json" -and $ltsRelease) { - $releaseTag = $buildInfo.ReleaseTag - $version = $releaseTag -replace '^v' - $semVersion = [System.Management.Automation.SemanticVersion] $version - - if ($semVersion.PreReleaseLabel -eq $null -and $semVersion.Minor % 2 -eq 0 -and $semVersion.Major -ge 7) - { - $ltsFile = "$ENV:PIPELINE_WORKSPACE/lts.json" - Copy-Item -Path $targetFile -Destination $ltsFile -Force - $vstsCommandLtsJsonFile = "vso[task.setvariable variable=LtsBuildInfoJsonFile]$ltsFile" - $vstsCommandCopyLTSBuildInfo = "vso[task.setvariable variable=CopyLTSBuildInfo]YES" - } + $ltsFile = "$ENV:PIPELINE_WORKSPACE/lts.json" + Copy-Item -Path $targetFile -Destination $ltsFile -Force + $vstsCommandLtsJsonFile = "vso[task.setvariable variable=LtsBuildInfoJsonFile]$ltsFile" + $vstsCommandCopyLTSBuildInfo = "vso[task.setvariable variable=CopyLTSBuildInfo]YES" } Write-Host "sending " + $vstsCommandCopyLTSBuildInfo @@ -69,6 +69,7 @@ steps: Destination: AzureBlob storage: '$(StorageAccount)' ContainerName: BuildInfo + condition: and(succeeded(), eq(variables['CopyStableBuildInfo'], 'YES')) - task: AzureFileCopy@2 displayName: 'AzureBlob build info ''lts.json'' Copy when needed' From df081aa1c13be9c82f8732c3773718555d94d32f Mon Sep 17 00:00:00 2001 From: travis plunk Date: Tue, 8 Mar 2022 12:33:13 -0800 Subject: [PATCH 20/38] Update parent job names due to merge conflict with 16641 and other conflicts --- .../releaseBuild/azureDevOps/releaseBuild.yml | 24 +++++++++---------- .../azureDevOps/templates/mac.yml | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index c6a759d1395..9f7b6c0b7d0 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -120,42 +120,42 @@ stages: - template: templates/windows-packaging.yml parameters: Architecture: x64 - parentJob: build_windows_x64 + parentJob: build_windows_x64_release - template: templates/windows-packaging.yml parameters: Architecture: x86 - parentJob: build_windows_x86 + parentJob: build_windows_x86_release - template: templates/windows-packaging.yml parameters: Architecture: arm - parentJob: build_windows_arm + parentJob: build_windows_arm_release - template: templates/windows-packaging.yml parameters: Architecture: arm64 - parentJob: build_windows_arm64 + parentJob: build_windows_arm64_release - template: templates/windows-packaging.yml parameters: Architecture: fxdependent - parentJob: build_windows_fxdependent + parentJob: build_windows_fxdependent_release - template: templates/windows-packaging.yml parameters: Architecture: fxdependentWinDesktop - parentJob: build_windows_fxdependentWinDesktop + parentJob: build_windows_fxdependentWinDesktop_release - template: templates/windows-package-signing.yml parameters: parentJobs: - - sign_windows_x64 - - sign_windows_x86 - - sign_windows_arm - - sign_windows_arm64 - - sign_windows_fxdependent - - sign_windows_fxdependentWinDesktop + - sign_windows_x64_release + - sign_windows_x86_release + - sign_windows_arm_release + - sign_windows_arm64_release + - sign_windows_fxdependent_release + - sign_windows_fxdependentWinDesktop_release - stage: compliance dependsOn: ['windows'] diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index 8815c142ddf..f3ef04a4c2a 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -50,7 +50,7 @@ jobs: $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' # Add -SkipReleaseChecks as a mitigation to unblock release. # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks + $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults -SkipReleaseChecks $env:AzDevOpsFeedPAT2 = $null displayName: 'Build' From 4d8a35a54e72b2553260fc7ceb01b8568ac8eab6 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Thu, 28 Oct 2021 20:51:35 -0700 Subject: [PATCH 21/38] Update feed and analyzer dependency (#16327) # Conflicts: # Analyzers.props --- .../azureDevOps/templates/insert-nuget-config-azfeed.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml index 5fdb9f7d977..9cf3d8dbc8d 100644 --- a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml +++ b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml @@ -15,11 +15,11 @@ steps: condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], '')) - pwsh: | Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsPackageFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination '${{ parameters.repoRoot }}' + New-NugetConfigFile -NugetFeedUrl $(PSInternalNugetFeed) -UserName $(PSInternalNugetFeedUserName) -ClearTextPAT $(PSInternalNugetFeedPAT) -FeedName AzDevOpsFeed -Destination '${{ parameters.repoRoot }}' if(-not (Test-Path "${{ parameters.repoRoot }}/nuget.config")) { throw "nuget.config is not created" } displayName: 'Add nuget.config for Azure DevOps feed for packages' - condition: and(succeededOrFailed(), ne(variables['AzDevOpsPackageFeed'], '')) + condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], '')) From 6a15f73b0a6b4eb06ba19bf6e52ce6550d5a3f0d Mon Sep 17 00:00:00 2001 From: travis plunk Date: Tue, 8 Mar 2022 16:51:47 -0800 Subject: [PATCH 22/38] Enable ARM64 packaging for macOS (#15768) # Conflicts: # tools/packaging/packaging.psm1 # tools/releaseBuild/azureDevOps/templates/mac-package-build.yml # tools/releaseBuild/azureDevOps/templates/mac.yml # Conflicts: # tools/ci.psm1 --- tools/ci.psm1 | 4 ++-- .../macOS/PowerShellPackageVsts.ps1 | 24 +++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/tools/ci.psm1 b/tools/ci.psm1 index 845462b841d..453c194b11b 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -14,8 +14,8 @@ if(Test-Path $dotNetPath) } # import build into the global scope so it can be used by packaging -Import-Module (Join-Path $repoRoot 'build.psm1') -Scope Global -Import-Module (Join-Path $repoRoot 'tools\packaging') -scope Global +Import-Module (Join-Path $repoRoot 'build.psm1') -Verbose -Scope Global +Import-Module (Join-Path $repoRoot 'tools\packaging') -Verbose -Scope Global # import the windows specific functcion only in Windows PowerShell or on Windows if($PSVersionTable.PSEdition -eq 'Desktop' -or $isWindows) diff --git a/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 b/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 index e2d6735bd4d..626ea07a93a 100644 --- a/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 +++ b/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 @@ -42,15 +42,18 @@ param ( [ValidatePattern("-signed.zip$")] [string]$BuildZip, - [string]$ArtifactName = 'result' + + [string]$ArtifactName = 'result', + + [switch]$SkipReleaseChecks ) $repoRoot = $location -if ($Build.IsPresent -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { - $releaseTagParam = @{ } +if ($Build -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { + $releaseTagParam = @{} if ($ReleaseTag) { - $releaseTagParam = @{ 'ReleaseTag' = $ReleaseTag } + $releaseTagParam['ReleaseTag'] = $ReleaseTag #Remove the initial 'v' from the ReleaseTag $version = $ReleaseTag -replace '^v' @@ -65,13 +68,14 @@ if ($Build.IsPresent -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { Push-Location try { + $pspackageParams = @{ SkipReleaseChecks = $SkipReleaseChecks; MacOSRuntime = $Runtime } Write-Verbose -Message "Init..." -Verbose Set-Location $repoRoot Import-Module "$repoRoot/build.psm1" Import-Module "$repoRoot/tools/packaging" Sync-PSTags -AddRemoteIfMissing - if ($BootStrap.IsPresent) { + if ($BootStrap) { Start-PSBootstrap -Package } @@ -81,15 +85,15 @@ try { Remove-Item -Path $BuildZip - Start-PSPackage @releaseTagParam + Start-PSPackage @pspackageParams @releaseTagParam switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @releaseTagParam } + "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam } } if ($LTS) { - Start-PSPackage @releaseTagParam -LTS + Start-PSPackage @pspackageParams @releaseTagParam -LTS switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @releaseTagParam -LTS } + "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam -LTS } } } } @@ -121,7 +125,7 @@ try { Pop-Location } -if ($Build.IsPresent -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { +if ($Build -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { $macPackages = Get-ChildItem "$repoRoot/powershell*" -Include *.pkg, *.tar.gz, *.zip foreach ($macPackage in $macPackages) { $filePath = $macPackage.FullName From 875b66aed39e957ce32bd91612ad2a35ad1f4263 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Tue, 8 Mar 2022 16:21:41 -0800 Subject: [PATCH 23/38] remove min-size linux build remove min-size # Conflicts: # PowerShell.Common.props # build.psm1 --- tools/packaging/packaging.psm1 | 18 +----------------- .../azureDevOps/templates/linux.yml | 10 ---------- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 0f4e63ceb03..5c8c271eb73 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -3897,14 +3897,6 @@ function Invoke-AzDevOpsLinuxPackageCreation { if ($BuildType -eq 'deb') { Start-PSPackage -Type tar @releaseTagParam -LTS:$LTS - Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${minSizeLinuxBuildFolder}-meta\psoptions.json" - - Write-Verbose -Verbose "---- Min-Size ----" - Write-Verbose -Verbose "options.Output: $($options.Output)" - Write-Verbose -Verbose "options.Top $($options.Top)" - - Start-PSPackage -Type min-size @releaseTagParam -LTS:$LTS - ## Create 'linux-arm' 'tar.gz' package. ## Note that 'linux-arm' can only be built on Ubuntu environment. Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${arm32LinuxBuildFolder}-meta\psoptions.json" @@ -3969,7 +3961,7 @@ function Invoke-AzDevOpsLinuxPackageBuild { if ($BuildType -eq 'deb') { ## Build 'min-size' $options = Get-PSOptions - Write-Verbose -Verbose "---- Min-Size ----" + Write-Verbose -Verbose "---- other builds ----" Write-Verbose -Verbose "options.Output: $($options.Output)" Write-Verbose -Verbose "options.Top $($options.Top)" $binDir = Join-Path -Path $options.Top -ChildPath 'bin' @@ -3978,14 +3970,6 @@ function Invoke-AzDevOpsLinuxPackageBuild { Remove-Item -Path $binDir -Recurse -Force } - $buildParams['Crossgen'] = $false - $buildParams['ForMinimalSize'] = $true - $buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${minSizeLinuxBuildFolder}" - Start-PSBuild -Clean @buildParams @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json" - # Remove symbol files, xml document files. - Remove-Item "${buildFolder}\*.pdb", "${buildFolder}\*.xml" -Force - - ## Build 'linux-arm' and create 'tar.gz' package for it. ## Note that 'linux-arm' can only be built on Ubuntu environment. $buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${arm32LinuxBuildFolder}" diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 19e0b4ca2dd..06abb34d108 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -80,16 +80,6 @@ jobs: PackageVersion: $(Version) sourceScanPath: '$(PowerShellRoot)/tools' - - ${{ if eq(variables.build,'deb') }} : - - template: Sbom.yml@ComplianceRepo - parameters: - BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildMinSize' - Build_Repository_Uri: $(Github_Build_Repository_Uri) - displayName: MinSize SBOM - PackageName: PowerShell Linux Minimum Size - PackageVersion: $(Version) - sourceScanPath: '$(PowerShellRoot)/tools' - - ${{ if eq(variables.build,'deb') }} : - template: Sbom.yml@ComplianceRepo parameters: From 861599800167546f7799716f8463f9e2fd715510 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 9 Mar 2022 13:10:37 -0800 Subject: [PATCH 24/38] Remove the platform check for rpm packages --- tools/packaging/packaging.psm1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 5c8c271eb73..1e3fe9e1de1 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -797,9 +797,6 @@ function New-UnixPackage { } $packageVersion = Get-LinuxPackageSemanticVersion -Version $Version - if (!$Environment.IsRedHatFamily -and !$Environment.IsSUSEFamily) { - throw ($ErrorMessage -f "Redhat or SUSE Family") - } } "osxpkg" { $packageVersion = $Version From c541579344973572704a382c690acdce36809ea2 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 9 Mar 2022 13:58:31 -0800 Subject: [PATCH 25/38] Fix test-distribution function to work with 7.1 --- tools/packaging/packaging.psm1 | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 1e3fe9e1de1..6fb203ea7b6 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1239,18 +1239,15 @@ function Test-Distribution throw "$Distribution is required for a Debian based distribution." } - if ( $Environment.IsDebianFamily -and $Script:DebianDistributions -notcontains $Distribution) - { - throw "$Distribution should be one of the following: $Script:DebianDistributions" - } - - if ( $Environment.IsRedHatFamily -and $Script:RedHatDistributions -notcontains $Distribution) - { - throw "$Distribution should be one of the following: $Script:RedHatDistributions" + if ($Script:DebianDistributions -notcontains $Distribution -and + $Script:RedHatDistributions -notcontains $Distribution -and + $Distribution -ne 'macOS') { + throw "$Distribution should be one of the following: $Script:DebianDistributions $Script:RedHatDistributions" } return $true } + function Get-PackageDependencies { param( From 9a108a9cc4ff58e8c59e938fdaf506c558429693 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Thu, 10 Mar 2022 11:13:41 -0800 Subject: [PATCH 26/38] Remove .dotnet folder --- .../releaseBuild/azureDevOps/templates/mac-package-build.yml | 4 ++++ tools/releaseBuild/azureDevOps/templates/mac.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index c792ac1c322..f636fbba363 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -108,6 +108,10 @@ jobs: Write-Host "##$vstsCommandString" displayName: Compress signed files + - pwsh: | + sudo pwsh -c 'Remove-Item /Users/runner/.dotnet -Force -Recurse' + displayName: Remove /Users/runner/.dotnet + - pwsh: | tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap displayName: 'Bootstrap VM' diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index f3ef04a4c2a..ec34762cc7b 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -38,6 +38,10 @@ jobs: parameters: nativePathRoot: '$(Agent.TempDirectory)' + - pwsh: | + sudo pwsh -c 'Remove-Item /Users/runner/.dotnet -Force -Recurse' + displayName: Remove /Users/runner/.dotnet + - pwsh: | tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap displayName: 'Bootstrap VM' From 13ea9d3c5a7fa154e22cd137b9fd455c5b1609a6 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Thu, 10 Mar 2022 11:43:18 -0800 Subject: [PATCH 27/38] Add --output only if it is not already present # Conflicts: # build.psm1 --- build.psm1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build.psm1 b/build.psm1 index 27bdbd29543..339e9c254a4 100644 --- a/build.psm1 +++ b/build.psm1 @@ -478,8 +478,10 @@ Fix steps: try { Push-Location $globalToolSrcFolder - $Arguments += "--output", $publishPath - Write-Log "Run dotnet $Arguments from $pwd to build global tool entry point" + if ($Arguments -notcontains '--output') { + $Arguments += "--output", $publishPath + } + Write-Log -message "Run dotnet $Arguments from $PWD to build global tool entry point" Start-NativeExecution { dotnet $Arguments } } finally { From 336321a8256c73476031f316bfe2d3332ac948db Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Thu, 10 Mar 2022 11:50:40 -0800 Subject: [PATCH 28/38] Pass SkipReleaseChecks for macOS packaging --- tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 b/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 index 626ea07a93a..8c867a8058b 100644 --- a/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 +++ b/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 @@ -68,7 +68,7 @@ if ($Build -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { Push-Location try { - $pspackageParams = @{ SkipReleaseChecks = $SkipReleaseChecks; MacOSRuntime = $Runtime } + $pspackageParams = @{ SkipReleaseChecks = $SkipReleaseChecks } Write-Verbose -Message "Init..." -Verbose Set-Location $repoRoot Import-Module "$repoRoot/build.psm1" @@ -101,22 +101,21 @@ try { if ($Build.IsPresent) { if ($Symbols.IsPresent) { Start-PSBuild -Configuration 'Release' -Crossgen -NoPSModuleRestore @releaseTagParam - $pspackageParams = @{} $pspackageParams['Type']='zip' $pspackageParams['IncludeSymbols']=$Symbols.IsPresent Write-Verbose "Starting powershell packaging(zip)..." -Verbose Start-PSPackage @pspackageParams @releaseTagParam } else { Start-PSBuild -Configuration 'Release' -Crossgen -PSModuleRestore @releaseTagParam - Start-PSPackage @releaseTagParam + Start-PSPackage @pspackageParams @releaseTagParam switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @releaseTagParam } + "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam } } if ($LTS) { Start-PSPackage @releaseTagParam -LTS switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @releaseTagParam -LTS } + "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam -LTS } } } } From 390962c5e08ebc92ee93591cddcf1aef142f929e Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 29 Oct 2021 11:18:43 -0700 Subject: [PATCH 29/38] Fix issues in release build (#16332) --- tools/packaging/packaging.psm1 | 12 ++++++++++-- .../azureDevOps/templates/compliance.yml | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 6fb203ea7b6..87fa2c41b87 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -3708,8 +3708,16 @@ function New-GlobalToolNupkg $packageInfo = @() - Remove-Item -Path (Join-Path $LinuxBinPath 'libcrypto.so.1.0.0') -Verbose -Force -Recurse - Remove-Item -Path (Join-Path $LinuxBinPath 'libssl.so.1.0.0') -Verbose -Force -Recurse + $libCryptoPath = Join-Path $LinuxBinPath 'libcrypto.so.1.0.0' + $libSSLPath = Join-Path $LinuxBinPath 'libssl.so.1.0.0' + + if (Test-Path $libCryptoPath) { + Remove-Item -Path $libCryptoPath -Verbose -Force + } + + if (Test-Path $libSSLPath) { + Remove-Item -Path $libSSLPath -Verbose -Force + } ## Remove unnecessary xml files Get-ChildItem -Path $LinuxBinPath, $WindowsBinPath, $WindowsDesktopBinPath -Filter *.xml | Remove-Item -Verbose diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 6d72f98c798..645a5302612 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -133,9 +133,13 @@ jobs: inputs: tsaVersion: TsaV2 codeBaseName: 'PowerShellCore_201906' + uploadAPIScan: false + uploadBinSkim: true + uploadCredScan: false uploadFortifySCA: false uploadFxCop: false uploadModernCop: false + uploadPoliCheck: false uploadPREfast: false uploadRoslyn: false uploadTSLint: false From 6d344b487a63e40eb7da04e8eabb755029298ca9 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 11 Mar 2022 10:29:41 -0800 Subject: [PATCH 30/38] Fix rpm specific issues in packaging --- tools/packaging/packaging.psm1 | 79 +++++++++++++++------------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 87fa2c41b87..a1828ce7774 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1187,7 +1187,8 @@ function Get-FpmArguments "-t", $Type, "-s", "dir" ) - if ($Environment.IsRedHatFamily) { + + if ($Type -eq "rpm") { $Arguments += @("--rpm-dist", $Distribution) $Arguments += @("--rpm-os", "linux") } @@ -1259,7 +1260,7 @@ function Get-PackageDependencies End { # These should match those in the Dockerfiles, but exclude tools like Git, which, and curl $Dependencies = @() - if ($Environment.IsDebianFamily) { + if ($Distribution -in $Script:DebianDistributions) { $Dependencies = @( "libc6", "libgcc1", @@ -1271,16 +1272,11 @@ function Get-PackageDependencies "libssl1.1|libssl1.0.2|libssl1.0.0" ) - } elseif ($Environment.IsRedHatFamily) { + } elseif ($Distribution -in $Script:RedHatDistributions) { $Dependencies = @( "openssl-libs", "libicu" ) - } elseif ($Environment.IsSUSEFamily) { - $Dependencies = @( - "libopenssl1_0_0", - "libicu" - ) } return $Dependencies @@ -1327,13 +1323,13 @@ function New-AfterScripts Write-Verbose -Message "AfterScript Distribution: $Distribution" -Verbose - if ($Environment.IsRedHatFamily) { + if ($Distribution -in $script:RedhatDistributions) { $AfterInstallScript = [io.path]::GetTempFileName() $AfterRemoveScript = [io.path]::GetTempFileName() $packagingStrings.RedHatAfterInstallScript -f "$Link" | Out-File -FilePath $AfterInstallScript -Encoding ascii $packagingStrings.RedHatAfterRemoveScript -f "$Link" | Out-File -FilePath $AfterRemoveScript -Encoding ascii } - elseif ($Environment.IsDebianFamily -or $Environment.IsSUSEFamily) { + elseif ($Distribution -in $Script:DebianDistributions) { $AfterInstallScript = [io.path]::GetTempFileName() $AfterRemoveScript = [io.path]::GetTempFileName() $packagingStrings.UbuntuAfterInstallScript -f "$Link" | Out-File -FilePath $AfterInstallScript -Encoding ascii @@ -1366,42 +1362,35 @@ function New-PSSymbolicLinks Write-Verbose -Message "PSSymLinks-Distribution: $Distribution" -Verbose - if ($Environment.IsRedHatFamily) { - switch -regex ($Distribution) - { - # add two symbolic links to system shared libraries that libmi.so is dependent on to handle - # platform specific changes. This is the only set of platforms needed for this currently - # as Ubuntu has these specific library files in the platform and macOS builds for itself - # against the correct versions. - 'centos\.8' { - New-Item -Force -ItemType SymbolicLink -Target "/lib64/libssl.so.1.1" -Path "$Staging/libssl.so.1.0.0" > $null - New-Item -Force -ItemType SymbolicLink -Target "/lib64/libcrypto.so.1.1.1" -Path "$Staging/libcrypto.so.1.0.0" > $null - } - default { - New-Item -Force -ItemType SymbolicLink -Target "/lib64/libssl.so.10" -Path "$Staging/libssl.so.1.0.0" > $null - New-Item -Force -ItemType SymbolicLink -Target "/lib64/libcrypto.so.10" -Path "$Staging/libcrypto.so.1.0.0" > $null - } + switch -regex ($Distribution) { + # add two symbolic links to system shared libraries that libmi.so is dependent on to handle + # platform specific changes. This is the only set of platforms needed for this currently + # as Ubuntu has these specific library files in the platform and macOS builds for itself + # against the correct versions. + 'centos\.8' { + New-Item -Force -ItemType SymbolicLink -Target "/lib64/libssl.so.1.1" -Path "$Staging/libssl.so.1.0.0" > $null + New-Item -Force -ItemType SymbolicLink -Target "/lib64/libcrypto.so.1.1.1" -Path "$Staging/libcrypto.so.1.0.0" > $null + } + 'rhel\.7' { + New-Item -Force -ItemType SymbolicLink -Target "/lib64/libssl.so.10" -Path "$Staging/libssl.so.1.0.0" > $null + New-Item -Force -ItemType SymbolicLink -Target "/lib64/libcrypto.so.10" -Path "$Staging/libcrypto.so.1.0.0" > $null + } + # add two symbolic links to system shared libraries that libmi.so is dependent on to handle + # platform specific changes. This appears to be a change in Debian 9; Debian 8 did not need these + # symlinks. + 'debian\.9' { + New-Item -Force -ItemType SymbolicLink -Target "/usr/lib/x86_64-linux-gnu/libssl.so.1.0.2" -Path "$Staging/libssl.so.1.0.0" > $null + New-Item -Force -ItemType SymbolicLink -Target "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.2" -Path "$Staging/libcrypto.so.1.0.0" > $null + } + 'debian\.(10|11)' { + New-Item -Force -ItemType SymbolicLink -Target "/usr/lib/x86_64-linux-gnu/libssl.so.1.1" -Path "$Staging/libssl.so.1.0.0" > $null + New-Item -Force -ItemType SymbolicLink -Target "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1" -Path "$Staging/libcrypto.so.1.0.0" > $null } - } - elseif ($Environment.IsDebianFamily -or $Environment.IsSUSEFamily) { - switch -regex ($Distribution) - { - # add two symbolic links to system shared libraries that libmi.so is dependent on to handle - # platform specific changes. This appears to be a change in Debian 9; Debian 8 did not need these - # symlinks. - 'debian\.9' { - New-Item -Force -ItemType SymbolicLink -Target "/usr/lib/x86_64-linux-gnu/libssl.so.1.0.2" -Path "$Staging/libssl.so.1.0.0" > $null - New-Item -Force -ItemType SymbolicLink -Target "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.2" -Path "$Staging/libcrypto.so.1.0.0" > $null - } - 'debian\.(10|11)' { - New-Item -Force -ItemType SymbolicLink -Target "/usr/lib/x86_64-linux-gnu/libssl.so.1.1" -Path "$Staging/libssl.so.1.0.0" > $null - New-Item -Force -ItemType SymbolicLink -Target "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1" -Path "$Staging/libcrypto.so.1.0.0" > $null - } - default { - # Default to old behavior before this change - New-Item -Force -ItemType SymbolicLink -Target "/lib64/libssl.so.10" -Path "$Staging/libssl.so.1.0.0" > $null - New-Item -Force -ItemType SymbolicLink -Target "/lib64/libcrypto.so.10" -Path "$Staging/libcrypto.so.1.0.0" > $null - } + default { + # Default to old behavior before this change + Write-Verbose -Verbose -Message "Default branch choosen for New-PSSymbolicLinks switch" + New-Item -Force -ItemType SymbolicLink -Target "/lib64/libssl.so.10" -Path "$Staging/libssl.so.1.0.0" > $null + New-Item -Force -ItemType SymbolicLink -Target "/lib64/libcrypto.so.10" -Path "$Staging/libcrypto.so.1.0.0" > $null } } } From 7ea6971cf739eb78b3da7984d3806461cbc44dc7 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 11 Mar 2022 12:06:01 -0800 Subject: [PATCH 31/38] Update findMissingNotices.ps1 to work with .NET 3.1 --- tools/findMissingNotices.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/findMissingNotices.ps1 b/tools/findMissingNotices.ps1 index 091acb2ce2a..fbff3a6f630 100644 --- a/tools/findMissingNotices.ps1 +++ b/tools/findMissingNotices.ps1 @@ -177,8 +177,8 @@ function Get-CGRegistrations { $registrationChanged = $false - $dotnetTargetName = 'net5.0' - $dotnetTargetNameWin7 = 'net5.0-windows7.0' + $dotnetTargetName = '.NETCoreApp,Version=v3.1' + $dotnetTargetNameWin7 = '.NETCoreApp,Version=v3.1' $unixProjectName = 'powershell-unix' $windowsProjectName = 'powershell-win-core' $actualRuntime = $Runtime @@ -264,7 +264,7 @@ function Get-CGRegistrations { $registrations = [System.Collections.Generic.Dictionary[string, Registration]]::new() $lastCount = 0 $registrationChanged = $false -foreach ($runtime in "win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win-arm", "linux-arm", "linux-arm64", "osx-arm64", "win-arm64", "win7-x86") { +foreach ($runtime in "win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win7-x86") { $registrationChanged = (Get-CGRegistrations -Runtime $runtime -RegistrationTable $registrations) -or $registrationChanged $count = $registrations.Count $newCount = $count - $lastCount From 9877b8d0d720074a8f6c81fc743f826753b70156 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 11 Mar 2022 12:10:04 -0800 Subject: [PATCH 32/38] Update packages and cgmanifest.json --- ...crosoft.PowerShell.Commands.Utility.csproj | 2 +- tools/cgmanifest.json | 118 ++++++++---------- 2 files changed, 55 insertions(+), 65 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj index da9e579de09..c8f69a2ff6c 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj +++ b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj @@ -33,7 +33,7 @@ - + diff --git a/tools/cgmanifest.json b/tools/cgmanifest.json index c0db3a5aa94..6c9a7c772e3 100644 --- a/tools/cgmanifest.json +++ b/tools/cgmanifest.json @@ -5,7 +5,7 @@ "Type": "nuget", "Nuget": { "Name": "Markdig.Signed", - "Version": "0.21.1" + "Version": "0.18.3" } }, "DevelopmentDependency": false @@ -15,7 +15,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.ApplicationInsights", - "Version": "2.15.0" + "Version": "2.13.1" } }, "DevelopmentDependency": false @@ -25,7 +25,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.CodeAnalysis.Analyzers", - "Version": "3.0.0" + "Version": "2.9.6" } }, "DevelopmentDependency": true @@ -35,7 +35,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.CodeAnalysis.Common", - "Version": "3.7.0" + "Version": "3.4.0" } }, "DevelopmentDependency": false @@ -45,7 +45,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.CodeAnalysis.CSharp", - "Version": "3.7.0" + "Version": "3.4.0" } }, "DevelopmentDependency": false @@ -95,7 +95,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.NETCore.Platforms", - "Version": "5.0.4" + "Version": "3.1.9" } }, "DevelopmentDependency": false @@ -125,7 +125,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.PowerShell.Native", - "Version": "7.1.0" + "Version": "7.0.0" } }, "DevelopmentDependency": false @@ -135,7 +135,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Win32.Registry.AccessControl", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -145,7 +145,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Win32.Registry", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -155,7 +155,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Win32.SystemEvents", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -165,7 +165,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Windows.Compatibility", - "Version": "5.0.0" + "Version": "3.1.2" } }, "DevelopmentDependency": false @@ -175,7 +175,7 @@ "Type": "nuget", "Nuget": { "Name": "Namotion.Reflection", - "Version": "1.0.14" + "Version": "1.0.13" } }, "DevelopmentDependency": false @@ -195,7 +195,7 @@ "Type": "nuget", "Nuget": { "Name": "NJsonSchema", - "Version": "10.2.2" + "Version": "10.1.26" } }, "DevelopmentDependency": false @@ -355,7 +355,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-arm.runtime.native.System.IO.Ports", - "Version": "5.0.0-rtm.20519.4" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -365,7 +365,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-arm64.runtime.native.System.IO.Ports", - "Version": "5.0.0-rtm.20519.4" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -375,7 +375,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-x64.runtime.native.System.IO.Ports", - "Version": "5.0.0-rtm.20519.4" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -395,7 +395,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.native.System.IO.Ports", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -445,7 +445,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.osx-x64.runtime.native.System.IO.Ports", - "Version": "5.0.0-rtm.20519.4" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -615,7 +615,7 @@ "Type": "nuget", "Nuget": { "Name": "System.CodeDom", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -645,7 +645,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ComponentModel.Composition.Registration", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -655,7 +655,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ComponentModel.Composition", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -665,7 +665,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Configuration.ConfigurationManager", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -685,7 +685,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Data.Odbc", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -695,7 +695,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Data.OleDb", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -735,7 +735,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Diagnostics.EventLog", - "Version": "5.0.1" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -745,7 +745,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Diagnostics.PerformanceCounter", - "Version": "5.0.1" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -755,7 +755,7 @@ "Type": "nuget", "Nuget": { "Name": "System.DirectoryServices.AccountManagement", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -765,7 +765,7 @@ "Type": "nuget", "Nuget": { "Name": "System.DirectoryServices.Protocols", - "Version": "5.0.1" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -775,7 +775,7 @@ "Type": "nuget", "Nuget": { "Name": "System.DirectoryServices", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -785,7 +785,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Drawing.Common", - "Version": "5.0.2" + "Version": "4.7.3" } }, "DevelopmentDependency": false @@ -800,16 +800,6 @@ }, "DevelopmentDependency": false }, - { - "Component": { - "Type": "nuget", - "Nuget": { - "Name": "System.Formats.Asn1", - "Version": "5.0.0" - } - }, - "DevelopmentDependency": false - }, { "Component": { "Type": "nuget", @@ -825,7 +815,7 @@ "Type": "nuget", "Nuget": { "Name": "System.IO.FileSystem.AccessControl", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -835,7 +825,7 @@ "Type": "nuget", "Nuget": { "Name": "System.IO.Packaging", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -845,7 +835,7 @@ "Type": "nuget", "Nuget": { "Name": "System.IO.Pipes.AccessControl", - "Version": "5.0.0" + "Version": "4.5.1" } }, "DevelopmentDependency": false @@ -855,7 +845,7 @@ "Type": "nuget", "Nuget": { "Name": "System.IO.Ports", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -895,7 +885,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Management", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -905,7 +895,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Memory", - "Version": "4.5.4" + "Version": "4.5.3" } }, "DevelopmentDependency": false @@ -915,7 +905,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Net.Http.WinHttpHandler", - "Version": "5.0.0" + "Version": "4.7.2" } }, "DevelopmentDependency": false @@ -955,7 +945,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Reflection.Context", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -965,7 +955,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Reflection.DispatchProxy", - "Version": "4.5.1" + "Version": "4.5.0" } }, "DevelopmentDependency": false @@ -1065,7 +1055,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Runtime.Caching", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -1075,7 +1065,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Runtime.CompilerServices.Unsafe", - "Version": "5.0.0" + "Version": "4.7.1" } }, "DevelopmentDependency": false @@ -1125,7 +1115,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.AccessControl", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -1135,7 +1125,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.Cng", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -1145,7 +1135,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.Pkcs", - "Version": "5.0.1" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -1155,7 +1145,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.ProtectedData", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -1165,7 +1155,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.Xml", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -1175,7 +1165,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Permissions", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -1185,7 +1175,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Principal.Windows", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -1245,7 +1235,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ServiceModel.Syndication", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -1255,7 +1245,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ServiceProcess.ServiceController", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -1265,7 +1255,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Text.Encoding.CodePages", - "Version": "5.0.0" + "Version": "4.7.1" } }, "DevelopmentDependency": false @@ -1285,7 +1275,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Text.Encodings.Web", - "Version": "5.0.1" + "Version": "4.7.2" } }, "DevelopmentDependency": false @@ -1295,7 +1285,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Threading.AccessControl", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false @@ -1335,7 +1325,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Windows.Extensions", - "Version": "5.0.0" + "Version": "4.7.0" } }, "DevelopmentDependency": false From e7cbabc2daea2316f4f05f0b7bf8ce7d4f33c07f Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 11 Mar 2022 13:19:42 -0800 Subject: [PATCH 33/38] Update wix file and global json --- assets/files.wxs | 24 ++++++++++++++++-------- global.json | 2 +- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/assets/files.wxs b/assets/files.wxs index 37113d2e2cb..3bbf92ce601 100644 --- a/assets/files.wxs +++ b/assets/files.wxs @@ -127,9 +127,6 @@ - - - @@ -1183,9 +1180,6 @@ - - - @@ -3017,6 +3011,19 @@ + + + + + + + + + + + + + @@ -3062,7 +3069,6 @@ - @@ -3414,7 +3420,6 @@ - @@ -3988,6 +3993,9 @@ + + + diff --git a/global.json b/global.json index b54dd5dd39a..b3084a944cc 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "3.1.414" + "version": "3.1.417" } } From 144c5be8ec02d5aa8450431ca3cedfa54b61370e Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 11 Mar 2022 13:55:35 -0800 Subject: [PATCH 34/38] Update wix file --- assets/files.wxs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/assets/files.wxs b/assets/files.wxs index 3bbf92ce601..bb56cea12df 100644 --- a/assets/files.wxs +++ b/assets/files.wxs @@ -3008,9 +3008,6 @@ - - - @@ -3024,6 +3021,9 @@ + + + @@ -3992,10 +3992,10 @@ - + From 78928080de2b0c10cd8e08398adf4191a5360bfb Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 11 Mar 2022 13:36:33 -0800 Subject: [PATCH 35/38] Build MarkdownRender nuget package --- tools/packaging/packaging.psm1 | 9 +++++++++ .../azureDevOps/templates/nuget-pkg-sbom.yml | 1 + 2 files changed, 10 insertions(+) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index a1828ce7774..e3b00019d9a 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1740,6 +1740,7 @@ function New-FileDependencies 'Microsoft.PowerShell.Commands.Utility' { $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.MarkdownRender'), [tuple]::Create('version', $PackageVersion))) > $null foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) { @@ -1801,6 +1802,14 @@ function New-FileDependencies $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null } } + + 'Microsoft.PowerShell.MarkdownRender' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } } Write-Output $deps diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml index fe8c1a872ff..a23e3e9d368 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -13,6 +13,7 @@ parameters: - Microsoft.PowerShell.Commands.Utility.dll - Microsoft.PowerShell.ConsoleHost.dll - Microsoft.PowerShell.CoreCLR.Eventing.dll + - Microsoft.PowerShell.MarkdownRender.dll - Microsoft.PowerShell.Security.dll - Microsoft.PowerShell.SDK.dll - Microsoft.WSMan.Management.dll From 2e3aad63d42ef0cd4cf58bf6060eda598f2ca780 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Mon, 29 Jun 2020 10:57:17 -0700 Subject: [PATCH 36/38] Fix NuGet package compliance issues (#13045) # Conflicts: # tools/packaging/packaging.psm1 --- tools/packaging/packaging.psm1 | 9 +++++++-- tools/packaging/packaging.strings.psd1 | 8 ++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index e3b00019d9a..dad70f0589f 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -8,6 +8,8 @@ $packagingStrings = Import-PowerShellDataFile "$PSScriptRoot\packaging.strings.p Import-Module "$PSScriptRoot\..\Xml" -ErrorAction Stop -Force $DebianDistributions = @("ubuntu.16.04", "ubuntu.18.04", "ubuntu.20.04", "debian.9", "debian.10", "debian.11") $RedhatDistributions = @("rhel.7","centos.8") +$script:iconFileName = "Powershell_black_64.png" +$script:iconPath = Join-Path -path $PSScriptRoot -ChildPath "../../assets/$iconFileName" -Resolve function Start-PSPackage { [CmdletBinding(DefaultParameterSetName='Version',SupportsShouldProcess=$true)] @@ -2170,7 +2172,7 @@ function New-NuSpec { throw "New-NuSpec can be only executed on Windows platform." } - $nuspecTemplate = $packagingStrings.NuspecTemplate -f $PackageId,$PackageVersion + $nuspecTemplate = $packagingStrings.NuspecTemplate -f $PackageId,$PackageVersion,$iconFileName $nuspecObj = [xml] $nuspecTemplate if ( ($null -ne $Dependency) -and $Dependency.Count -gt 0 ) { @@ -3750,6 +3752,9 @@ function New-GlobalToolNupkg $packageInfo | ForEach-Object { $ridFolder = New-Item -Path (Join-Path $_.RootFolder "tools/netcoreapp3.1/any") -ItemType Directory + # Add the icon file to the package + Copy-Item -Path $iconPath -Destination "$($_.RootFolder)/$iconFileName" -Verbose + $packageType = $_.Type switch ($packageType) @@ -3836,7 +3841,7 @@ function New-GlobalToolNupkg } $packageName = $_.PackageName - $nuSpec = $packagingStrings.GlobalToolNuSpec -f $packageName, $PackageVersion + $nuSpec = $packagingStrings.GlobalToolNuSpec -f $packageName, $PackageVersion, $iconFileName $nuSpec | Out-File -FilePath (Join-Path $_.RootFolder "$packageName.nuspec") -Encoding ascii $toolSettings | Out-File -FilePath (Join-Path $ridFolder "DotnetToolSettings.xml") -Encoding ascii diff --git a/tools/packaging/packaging.strings.psd1 b/tools/packaging/packaging.strings.psd1 index 3327fc28663..d2ffee81ab4 100644 --- a/tools/packaging/packaging.strings.psd1 +++ b/tools/packaging/packaging.strings.psd1 @@ -130,11 +130,11 @@ open {0} {1} Microsoft Microsoft,PowerShell - true + false Runtime for hosting PowerShell https://github.com/PowerShell/PowerShell - https://github.com/PowerShell/PowerShell/blob/master/assets/Powershell_black_64.png?raw=true - https://github.com/PowerShell/PowerShell/blob/master/LICENSE.txt + {2} + MIT PowerShell en-US © Microsoft Corporation. All rights reserved. @@ -169,7 +169,7 @@ open {0} Microsoft Microsoft,PowerShell https://github.com/PowerShell/PowerShell - https://github.com/PowerShell/PowerShell/blob/master/assets/Powershell_black_64.png?raw=true + {2} false PowerShell global tool MIT From 6cf59adb1f1aba08808ca1efe05d685430a359d6 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Mon, 14 Mar 2022 21:50:05 +0000 Subject: [PATCH 37/38] Merged PR 19569: Fix the NuGet SDK package creation Fix the NuGet SDK package creation --- tools/packaging/packaging.psm1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index dad70f0589f..d5634a07dea 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1915,7 +1915,7 @@ function New-ILNugetPackageSource Write-Verbose -Verbose "New-ILNugetPackageSource: Creating package source folder for file: $FileName at: $filePackageFolder" #region ref - $refFolder = New-Item (Join-Path $filePackageFolder.FullName "ref/$script:netCoreRuntime") -ItemType Directory -Force + $refFolder = New-Item (Join-Path $filePackageFolder.FullName "ref/netcoreapp3.1") -ItemType Directory -Force CopyReferenceAssemblies -assemblyName $fileBaseName -refBinPath $refBinPath -refNugetPath $refFolder -assemblyFileList $fileList #endregion ref $packageRuntimesFolderPath = $packageRuntimesFolder.FullName @@ -1958,8 +1958,8 @@ function New-ILNugetPackageSource "Microsoft.PowerShell.Utility" ) - $winModuleFolder = New-Item (Join-Path $contentFolder "runtimes\win\lib\$script:netCoreRuntime\Modules") -ItemType Directory -Force - $unixModuleFolder = New-Item (Join-Path $contentFolder "runtimes\unix\lib\$script:netCoreRuntime\Modules") -ItemType Directory -Force + $winModuleFolder = New-Item (Join-Path $contentFolder "runtimes\win\lib\netcoreapp3.1\Modules") -ItemType Directory -Force + $unixModuleFolder = New-Item (Join-Path $contentFolder "runtimes\unix\lib\netcoreapp3.1\Modules") -ItemType Directory -Force foreach ($module in $winBuiltInModules) { $source = Join-Path $WinFxdBinPath "Modules\$module" From 3269c95a72af9a3a161884dbae99eae0352b3e7d Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Wed, 16 Mar 2022 17:35:06 +0000 Subject: [PATCH 38/38] Merged PR 19580: Update change log for v7.0.9 release Update change log for v7.0.9 release --- .spelling | 5 +++++ CHANGELOG/7.0.md | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/.spelling b/.spelling index 3cb3e18bc83..1a9c74aa14d 100644 --- a/.spelling +++ b/.spelling @@ -952,4 +952,9 @@ createdump vPack NoLanguage PkgES +buildinfo +SKUs +CGManifest +vmImage +ci.psm1 centos-7 diff --git a/CHANGELOG/7.0.md b/CHANGELOG/7.0.md index 4d01a181a36..aac6756c4e7 100644 --- a/CHANGELOG/7.0.md +++ b/CHANGELOG/7.0.md @@ -1,5 +1,45 @@ # 7.0 Changelog +## [7.0.9] - 2022-03-16 + +### Build and Packaging Improvements + +
+ + + +

Update .NET SDK to 3.1.417

+ + + +
    +
  • Fix the NuGet SDK package creation (Internal 19569)
    • +
  • Fix NuGet package compliance issues (#13045)
    • +
  • Fix issues in release build (#16332)
    • +
  • Enable ARM64 packaging for macOS (#15768)
    • +
  • Update feed and analyzer dependency (#16327)
    • +
  • Only upload stable buildinfo for stable releases (#16251)
    • +
  • Opt-in to build security monitoring (#16911)
    • +
  • Update experimental feature json files (#16838) (Thanks @!)
    • +
  • Ensure alpine and arm SKUs have the PowerShell configuration file with experimental features enabled (#16823)
    • +
  • Remove WiX install (#16834)
    • +
  • Add Linux package dependencies for packaging (#16807)
    • +
  • Switch to our custom images for build and release (#16801)
    • +
  • Remove all references to cmake for the builds in this repo (#16578)
    • +
  • Register NuGet source when generating CGManifest (#16570)
    • +
  • Update Images used for release (#16580)
    • +
  • Add Software Bill of Materials to the main packages (#16202, #16641, #16711)
    • +
  • Add GitHub Workflow to keep notices up to date (#16284)
    • +
  • Update the vmImage and PowerShell root directory for macOS builds (#16611)
    • +
  • Update macOS build image and root folder for build (#16609)
    • +
  • Add checkout to build json stage to get ci.psm1 (#16399)
    • +
  • Move mapping file into product repo and add Debian 11 (#16316)
    • +
+ +
+ +[7.0.9]: https://github.com/PowerShell/PowerShell/compare/v7.0.8...v7.0.9 + ## [7.0.8] - 2021-10-14 ### Engine Updates and Fixes