From 3d967137b0d6d160726034895a8591094fda9f29 Mon Sep 17 00:00:00 2001 From: Justin Chung <124807742+jshigetomi@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:41:00 -0600 Subject: [PATCH] Deploy Box Update (#24632) * reverted one time changes * added ob_outputDirectory * added mscode variable group * forgot to move psm1 to ToolArtifact folder * added pipeline name and passing toolartifact through context * added tools * PSPackagesOfficial * removed tools clone * added uploading CL * verbose statements for release tag and release version * trying to regerence variable with step name * checking REPOROOT * removed get child item * Deploy box update * nuget template context pipeline name correction * changed other instances of releasetag * added -Verbose to Copy-Item * checkout task, instead of git clone * changed to Build.SourcesDirectory * removed path specification * removed path * added tsa and credscan * not copying tools to artifacts * official and production --------- Co-authored-by: Justin Chung Co-authored-by: Travis Plunk --- .pipelines/PowerShell-Release-Official.yml | 19 +++-- .../release-SetReleaseTagandContainerName.yml | 6 +- .../templates/release-SetTagAndTools.yml | 75 +++++++++++++++++++ .pipelines/templates/release-githubtasks.yml | 53 ++++--------- .../templates/release-publish-nuget.yml | 15 ++-- .../release-validate-packagenames.yml | 2 +- .pipelines/templates/uploadToAzure.yml | 32 ++++---- .../templates/variable/release-shared.yml | 10 +++ 8 files changed, 136 insertions(+), 76 deletions(-) create mode 100644 .pipelines/templates/release-SetTagAndTools.yml diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 03defa5d128..6a17139e05e 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -63,6 +63,10 @@ resources: type: git name: OneBranch.Pipelines/GovernedTemplates ref: refs/heads/main + - repository: PSInternalTools + type: git + name: PowerShellCore/Internal-PowerShellTeam-Tools + ref: refs/heads/master pipelines: - pipeline: CoOrdinatedBuildPipeline @@ -110,17 +114,14 @@ extends: tsaOptionsFile: .config\tsaoptions.json stages: - - stage: DownloadPackages - displayName: 'Download Packages' - dependsOn: [] + - stage: setReleaseTagAndUploadTools + displayName: 'Set Release Tag and Upload Tools' jobs: - - template: /.pipelines/templates/release-download-packages.yml@self + - template: /.pipelines/templates/release-SetTagAndTools.yml@self - stage: msixbundle displayName: 'Create MSIX Bundle' dependsOn: [] - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/release-create-msix.yml@self @@ -278,7 +279,7 @@ extends: - stage: PublishGitHubRelease displayName: Publish GitHub Release dependsOn: - - DownloadPackages + - setReleaseTagAndUploadTools - UpdateChangeLog variables: ob_release_environment: Production @@ -316,7 +317,9 @@ extends: - stage: PublishNuGet displayName: Publish NuGet - dependsOn: PushGitTagAndMakeDraftPublic + dependsOn: + - setReleaseTagAndUploadTools + - PushGitTagAndMakeDraftPublic variables: ob_release_environment: Production jobs: diff --git a/.pipelines/templates/release-SetReleaseTagandContainerName.yml b/.pipelines/templates/release-SetReleaseTagandContainerName.yml index 7e88624b45c..667132f5f90 100644 --- a/.pipelines/templates/release-SetReleaseTagandContainerName.yml +++ b/.pipelines/templates/release-SetReleaseTagandContainerName.yml @@ -8,9 +8,10 @@ steps: } $releaseTag = $Branch -replace '^.*((release|rebuild)/)' - $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" + $vstsCommandString = "vso[task.setvariable variable=$Variable;isOutput=true]$releaseTag" Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose Write-Host -Object "##$vstsCommandString" + name: OutputReleaseTag displayName: Set Release Tag - pwsh: | @@ -20,7 +21,8 @@ steps: Write-Host "##$vstsCommandString" $version = '$(ReleaseTag)'.ToLowerInvariant().Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" + $vstsCommandString = "vso[task.setvariable variable=Version;isOutput=true]$version" Write-Host ("sending " + $vstsCommandString) Write-Host "##$vstsCommandString" + name: OutputVersion displayName: Set container name diff --git a/.pipelines/templates/release-SetTagAndTools.yml b/.pipelines/templates/release-SetTagAndTools.yml new file mode 100644 index 00000000000..7b8a946e323 --- /dev/null +++ b/.pipelines/templates/release-SetTagAndTools.yml @@ -0,0 +1,75 @@ +jobs: +- job: SetTagAndTools + displayName: Set Tag and Tools + condition: succeeded() + pool: + type: windows + variables: + - group: 'mscodehub-code-read-akv' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + steps: + - template: release-SetReleaseTagandContainerName.yml@self + + - checkout: self + clean: true + env: + ob_restore_phase: true + + - checkout: PSInternalTools + clean: true + env: + ob_restore_phase: true + + - pwsh: | + New-Item -ItemType Directory -Path '$(Pipeline.Workspace)/ToolArtifact' + Get-ChildItem -Path '$(Build.SourcesDirectory)/Internal-PowerShellTeam-Tools/Scripts' -Filter 'GitHubRelease.psm1' -ErrorAction SilentlyContinue | + Copy-Item -Destination '$(Pipeline.Workspace)/ToolArtifact' -Verbose + displayName: Move GitHub Tool + + - task: onebranch.pipeline.signing@1 + displayName: Sign Tools + inputs: + command: 'sign' + signing_profile: internal_azure_service + files_to_sign: '*.ps1;*.psm1' + search_root: '$(Pipeline.Workspace)/ToolArtifact' + + - pwsh: | + Write-Verbose -Verbose "Creating output directory for release tools: $(ob_outputDirectory)/ToolArtifact" + New-Item -Path $(ob_outputDirectory)/ToolArtifact -ItemType Directory -Force + Get-ChildItem -Path "$(Pipeline.Workspace)/ToolArtifact/*" -Recurse | + Copy-Item -Destination $(ob_outputDirectory)/ToolArtifact -Recurse -Verbose + displayName: Upload Tools + + - pwsh: | + Write-Verbose -Verbose "Release Tag: $(OutputReleaseTag.releaseTag)" + $releaseVersion = '$(OutputReleaseTag.releaseTag)' -replace '^v','' + Write-Verbose -Verbose "Release Version: $releaseVersion" + $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion + + $isPreview = $semanticVersion.PreReleaseLabel -ne $null + + $fileName = if ($isPreview) { + "preview.md" + } + else { + $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" + } + + $filePath = "$(Build.SourcesDirectory)/PowerShell/CHANGELOG/$fileName" + Write-Verbose -Verbose "Selected Log file: $filePath" + + if (-not (Test-Path -Path $filePath)) { + Write-Error "Changelog file not found: $filePath" + exit 1 + } + + Write-Verbose -Verbose "Creating output directory for CHANGELOG: $(ob_outputDirectory)/CHANGELOG" + New-Item -Path $(ob_outputDirectory)/CHANGELOG -ItemType Directory -Force + Copy-Item -Path $filePath -Destination $(ob_outputDirectory)/CHANGELOG + displayName: Upload Changelog diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index bfc500585b0..ed3ae028934 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -8,32 +8,26 @@ jobs: templateContext: inputs: - input: pipelineArtifact - artifactName: drop_DownloadPackages_upload_packages + artifactName: drop_setReleaseTagAndUploadTools_SetTagAndTools + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_upload_upload_packages variables: - - template: ./variable/release-shared.yml@self + - template: ./variable/release-shared.yml@self + parameters: + RELEASETAG: $[ stageDependencies.setReleaseTagAndUploadTools.SetTagAndTools.outputs['OutputReleaseTag.releaseTag'] ] steps: - task: PowerShell@2 inputs: targetType: inline script: | - Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose + Write-Verbose -Verbose "Release Tag: $(ReleaseTag)" + Get-ChildItem Env: | Out-String -Stream | Write-Verbose -Verbose displayName: 'Capture Environment Variables' - template: release-install-pwsh.yml - - template: release-checkout-pwsh-repo.yml - - - template: release-SetReleaseTagAndContainerName.yml - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - git clone --depth 1 https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' - displayName: Clone Internal-Tools repository - - task: PowerShell@2 inputs: targetType: inline @@ -55,17 +49,6 @@ jobs: Write-Verbose -Verbose -Message $fileContent displayName: Add sha256 hashes - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set release version' - - task: PowerShell@2 inputs: targetType: inline @@ -79,21 +62,11 @@ jobs: targetType: inline pwsh: true script: | - Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion - - $isPreview = $semanticVersion.PreReleaseLabel -ne $null - - $fileName = if ($isPreview) { - "preview.md" - } - else { - $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" - } + Import-module '$(Pipeline.Workspace)/ToolArtifact/GitHubRelease.psm1' + Write-Verbose -Verbose "Available modules: " + Get-Module | Write-Verbose -Verbose - $filePath = "$(Pipeline.Workspace)/PowerShell/CHANGELOG/$fileName" - Write-Verbose -Verbose "Selected Log file: $filePath" + $filePath = Get-ChildItem -Path "$(Pipeline.Workspace)/CHANGELOG" -Filter '*.md' | Select-Object -First 1 -ExpandProperty FullName if (-not (Test-Path $filePath)) { throw "$filePath not found" diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml index db0978e0b0b..78338d7d87c 100644 --- a/.pipelines/templates/release-publish-nuget.yml +++ b/.pipelines/templates/release-publish-nuget.yml @@ -13,20 +13,19 @@ jobs: templateContext: inputs: - input: pipelineArtifact - artifactName: drop_DownloadPackages_upload_packages - + pipeline: PSPackagesOfficial + artifactName: drop_upload_upload_packages variables: - template: ./variable/release-shared.yml@self + parameters: + VERSION: $[ stageDependencies.setReleaseTagAndUploadTools.SetTagAndTools.outputs['OutputVersion.Version'] ] steps: - template: release-install-pwsh.yml - - template: release-checkout-pwsh-repo.yml - - - template: release-SetReleaseTagAndContainerName.yml - - pwsh: | - Get-ChildItem Env: + Write-Verbose -Verbose "Version: $(Version)" + Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' - pwsh: | @@ -34,7 +33,7 @@ jobs: $null = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/release" Copy-Item "$(Pipeline.Workspace)/NuGetPackages/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose - $releaseVersion = '$(VERSION)' + $releaseVersion = '$(Version)' $globalToolPath = "$(Pipeline.Workspace)/NuGetPackages/PowerShell.$releaseVersion.nupkg" if ($releaseVersion -notlike '*-*') { diff --git a/.pipelines/templates/release-validate-packagenames.yml b/.pipelines/templates/release-validate-packagenames.yml index cadf0c1ba12..8b08f8d8436 100644 --- a/.pipelines/templates/release-validate-packagenames.yml +++ b/.pipelines/templates/release-validate-packagenames.yml @@ -23,7 +23,7 @@ jobs: displayName: Capture environment - pwsh: | - $name = "{0}_{1:x}" -f '$(releaseTag)', (Get-Date).Ticks + $name = "{0}_{1:x}" -f '$(OutputReleaseTag.releaseTag)', (Get-Date).Ticks Write-Host $name Write-Host "##vso[build.updatebuildnumber]$name" displayName: Set Release Name diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 50a4a676fee..35a11ec383c 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -235,23 +235,21 @@ jobs: Get-ChildItem '$(Build.ArtifactStagingDirectory)/downloads' | Select-Object -ExpandProperty FullName displayName: 'Capture downloads' - # - pwsh: | - # Write-Verbose -Verbose "Copying Github Release files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" - # - # Write-Verbose -Verbose "Creating output directory for GitHub Release files: $(ob_outputDirectory)/GitHubPackages" - # New-Item -Path $(ob_outputDirectory)/GitHubPackages -ItemType Directory -Force - # Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | - # Where-Object { $_.Extension -notin '.msix', '.nupkg' } | - # ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | - # Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse - # - # Write-Verbose -Verbose "Creating output directory for NuGet packages: $(ob_outputDirectory)/NuGetPackages" - # New-Item -Path $(ob_outputDirectory)/NuGetPackages -ItemType Directory -Force - # Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | - # Where-Object { $_.Extension -eq '.nupkg' } | - # ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | - # Copy-Item -Destination $(ob_outputDirectory)/NuGetPackages -Recurse - # displayName: Copy downloads to Artifacts + - pwsh: | + Write-Verbose -Verbose "Copying Github Release files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" + + Write-Verbose -Verbose "Creating output directory for GitHub Release files: $(ob_outputDirectory)/GitHubPackages" + New-Item -Path $(ob_outputDirectory)/GitHubPackages -ItemType Directory -Force + Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | + Where-Object { $_.Extension -notin '.msix', '.nupkg' } | + Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse -Verbose + + Write-Verbose -Verbose "Creating output directory for NuGet packages: $(ob_outputDirectory)/NuGetPackages" + New-Item -Path $(ob_outputDirectory)/NuGetPackages -ItemType Directory -Force + Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | + Where-Object { $_.Extension -eq '.nupkg' } | + Copy-Item -Destination $(ob_outputDirectory)/NuGetPackages -Recurse -Verbose + displayName: Copy downloads to Artifacts - pwsh: | # Create output directory for packages which have been uploaded to blob storage diff --git a/.pipelines/templates/variable/release-shared.yml b/.pipelines/templates/variable/release-shared.yml index 92ab56199d4..f944639a908 100644 --- a/.pipelines/templates/variable/release-shared.yml +++ b/.pipelines/templates/variable/release-shared.yml @@ -5,6 +5,12 @@ parameters: - name: SBOM type: boolean default: false + - name: RELEASETAG + type: string + default: 'Not Initialized' + - name: VERSION + type: string + default: 'Not Initialized' variables: - name: ob_signing_setup_enabled @@ -30,3 +36,7 @@ variables: value: ${{ parameters.REPOROOT }}\.config\suppress.json - name: ob_sdl_codeql_compiled_enabled value: false + - name: ReleaseTag + value: ${{ parameters.RELEASETAG }} + - name: Version + value: ${{ parameters.VERSION }}