From 62180f9aa03284d8d6961ac8e8a46f2096a6cbf6 Mon Sep 17 00:00:00 2001 From: Patrick Meinecke Date: Wed, 7 Aug 2024 14:45:46 -0400 Subject: [PATCH 1/7] Copy to static site instead of making blob public --- .../templates/release-MakeBlobPublic.yml | 114 ++++++++++-------- 1 file changed, 67 insertions(+), 47 deletions(-) diff --git a/.pipelines/templates/release-MakeBlobPublic.yml b/.pipelines/templates/release-MakeBlobPublic.yml index 2cfabb4fd4b..c0104ac4c2a 100644 --- a/.pipelines/templates/release-MakeBlobPublic.yml +++ b/.pipelines/templates/release-MakeBlobPublic.yml @@ -1,31 +1,22 @@ jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: - displayName: Approve Blob Public - jobName: ApproveBlobPublic + displayName: Approve Copy release packages to PSInfra storage + jobName: CopyReleaseBlobApproval instructions: | - Are you sure you want to make the blob public? + Approval for Copy release packages to PSInfra storage -- job: blobPublic - displayName: Make Azure Blob Public - dependsOn: ApproveBlobPublic - condition: succeeded() +- job: PSInfraReleaseBlobPublic + displayName: Copy release to PSInfra storage + dependsOn: CopyReleaseBlobApproval pool: type: windows + variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 + - group: 'PSInfraStorage' - group: 'Azure Blob variable group' - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_codeSignValidation_enabled - value: false - - name: ob_sdl_binskim_enabled - value: false - name: ob_sdl_tsa_configFile value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - name: ob_sdl_credscan_suppressionsFile @@ -34,47 +25,76 @@ jobs: value: false steps: - - checkout: self - clean: true - env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - - template: /.pipelines/templates/SetVersionVariables.yml@self - parameters: - ReleaseTagVar: $(ReleaseTagVar) - CreateJson: yes - UseJson: no + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no - pwsh: | Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' - - pwsh: | - $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose - if ($azureRmModule) { - Write-Host 'AzureRM module exists. Removing it' - Uninstall-AzureRm - Write-Host 'AzureRM module removed' - } - - Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose - displayName: Remove AzRM modules - - - task: AzureCLI@2 - displayName: 'Set blob permissions' - inputs: - azureSubscription: az-blob-cicd-infra - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - az storage container set-permission --account-name $(StorageAccount) --name $(azureVersion) --public-access blob - az storage container set-permission --account-name $(StorageAccount) --name $(azureVersion)-gc --public-access blob + - pwsh: | + $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose + if ($azureRmModule) { + Write-Host 'AzureRM module exists. Removing it' + Uninstall-AzureRm + Write-Host 'AzureRM module removed' + } + + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + displayName: Remove AzRM modules + + - task: AzurePowerShell@5 + displayName: Copy blobs to PSInfra storage + inputs: + azureSubscription: az-blob-cicd-infra + scriptType: inlineScript + azurePowerShellVersion: LatestVersion + pwsh: true + inline: | + $sourceStorageAccountName = '$(StorageAccount)' + $destinationStorageAccountName = '$(PSInfraStorageAccount)' + $destinationContainerName = '$web' + $destinationPrefix = '$(ReleaseTagVar)' + + $sourceContext = New-AzStorageContext -StorageAccountName $sourceStorageAccountName + Write-Verbose -Verbose "Source context: $($sourceContext.BlobEndPoint)" + + $destinationContext = New-AzStorageContext -StorageAccountName $destinationStorageAccountName + Write-Verbose -Verbose "Destination context: $($destinationContext.BlobEndPoint)" + + foreach ($sourceContainerName in '$(AzureVersion)', '$(AzureVersion)-gc') { + $blobs = Get-AzStorageBlob -Context $sourceContext -Container $sourceContainerName + + Write-Verbose -Verbose "Blobs found in $sourceContainerName" + $blobs.Name | Write-Verbose -Verbose + + Write-Verbose -Verbose "Copying blobs from $sourceContainerName to $destinationContainerName/$destinationPrefix" + + foreach ($blob in $blobs) { + $sourceBlobName = $blob.Name + Write-Verbose -Verbose "sourceBlobName = $sourceBlobName" + + $destinationBlobName = "$destinationPrefix/$sourceBlobName" + Write-Verbose -Verbose "destinationBlobName = $destinationBlobName" + + Copy-AzStorageBlob -SourceContext $sourceContext -DestinationContext $destinationContext -SrcContainer $sourceContainerName -SrcBlob $sourceBlobName -DestContainer $destinationContainerName -DestBlob $destinationBlobName -Force -Verbose -Confirm:$false + } + } + - template: /.pipelines/templates/approvalJob.yml@self parameters: displayName: Approve Copy Global tool packages to PSInfra storage jobName: CopyBlobApproval - dependsOnJob: blobPublic + dependsOnJob: PSInfraReleaseBlobPublic instructions: | Approval for Copy global tool packages to PSInfra storage From f08c91e44ac245924a9811e0476edc8b76cae775 Mon Sep 17 00:00:00 2001 From: Patrick Meinecke Date: Tue, 10 Sep 2024 15:14:16 -0400 Subject: [PATCH 2/7] Move buildinfo to static site --- .pipelines/templates/release-upload-buildinfo.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.pipelines/templates/release-upload-buildinfo.yml b/.pipelines/templates/release-upload-buildinfo.yml index 5696a9c0d22..f94c112d84a 100644 --- a/.pipelines/templates/release-upload-buildinfo.yml +++ b/.pipelines/templates/release-upload-buildinfo.yml @@ -118,8 +118,9 @@ jobs: azurePowerShellVersion: LatestVersion pwsh: true inline: | - $containerName = "buildinfo" - $storageAccount = '$(StorageAccount)' + $containerName = '$web' + $storageAccount = '$(PSInfraStorageAccount)' + $prefix = "buildinfo" $storageContext = New-AzStorageContext -StorageAccountName $storageAccount -UseConnectedAccount @@ -127,20 +128,20 @@ jobs: $jsonFile = "$env:BuildInfoJsonFile" $blobName = Get-Item $jsonFile | Split-Path -Leaf Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" - Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext -Force + Set-AzStorageBlobContent -File "$prefix/$jsonFile" -Container $containerName -Blob $blobName -Context $storageContext -Force } if ($env:CopyLTSBuildInfo -eq 'YES') { $jsonFile = "$env:LtsBuildInfoJsonFile" $blobName = Get-Item $jsonFile | Split-Path -Leaf Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" - Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext -Force + Set-AzStorageBlobContent -File "$prefix/$jsonFile" -Container $containerName -Blob $blobName -Context $storageContext -Force } if ($env:CopyVersionBuildInfo -eq 'YES') { $jsonFile = "$env:VersionBuildInfoJsonFile" $blobName = Get-Item $jsonFile | Split-Path -Leaf Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" - Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext -Force + Set-AzStorageBlobContent -File "$prefix/$jsonFile" -Container $containerName -Blob $blobName -Context $storageContext -Force } condition: and(succeeded(), eq(variables['CopyMainBuildInfo'], 'YES')) From 92390e2f02ebc1e1f06b367e9a2ffb2b5386b4cb Mon Sep 17 00:00:00 2001 From: Patrick Meinecke Date: Tue, 10 Sep 2024 15:14:43 -0400 Subject: [PATCH 3/7] Fix install-powershell.ps1 to look at static site --- tools/install-powershell.ps1 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/install-powershell.ps1 b/tools/install-powershell.ps1 index 2c9eb0d98f1..3cd5fd464f4 100644 --- a/tools/install-powershell.ps1 +++ b/tools/install-powershell.ps1 @@ -268,7 +268,6 @@ try { if ($Daily) { $metadata = Invoke-RestMethod 'https://aka.ms/pwsh-buildinfo-daily' $release = $metadata.ReleaseTag -replace '^v' - $blobName = $metadata.BlobName # Get version from currently installed PowerShell Daily if available. $pwshPath = if ($IsWinEnv) {Join-Path $Destination "pwsh.exe"} else {Join-Path $Destination "pwsh"} @@ -297,8 +296,7 @@ try { throw "The OS architecture is '$architecture'. However, we currently only support daily package for x64." } - - $downloadURL = "https://pscoretestdata.blob.core.windows.net/${blobName}/${packageName}" + $downloadURL = "https://powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net/$($metadata.ReleaseTag)/$packageName" Write-Verbose "About to download package from '$downloadURL'" -Verbose $packagePath = Join-Path -Path $tempDir -ChildPath $packageName From a7c07090d50cf6004198efffd94d375732070454 Mon Sep 17 00:00:00 2001 From: Patrick Meinecke Date: Mon, 23 Sep 2024 13:56:31 -0400 Subject: [PATCH 4/7] Update paths in buildinfo step --- .pipelines/templates/release-upload-buildinfo.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-upload-buildinfo.yml b/.pipelines/templates/release-upload-buildinfo.yml index f94c112d84a..d03de64da0d 100644 --- a/.pipelines/templates/release-upload-buildinfo.yml +++ b/.pipelines/templates/release-upload-buildinfo.yml @@ -45,14 +45,14 @@ jobs: displayName: Download build info artifact - pwsh: | - Import-Module '$(Build.SourcesDirectory)/tools/ci.psm1' + Import-Module '$(Build.SourcesDirectory)/PowerShell/tools/ci.psm1' $jsonFile = Get-Item "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/BuildInfoJson/*.json" $fileName = Split-Path $jsonFile -Leaf $dateTime = [datetime]::UtcNow $dateTime = [datetime]::new($dateTime.Ticks - ($dateTime.Ticks % [timespan]::TicksPerSecond), $dateTime.Kind) - $metadata = Get-Content ./tools/metadata.json | ConvertFrom-Json + $metadata = Get-Content -LiteralPath '$(Build.SourcesDirectory)/PowerShell/tools/metadata.json' -ErrorAction Stop | ConvertFrom-Json $stableRelease = $metadata.StableRelease.Latest $ltsRelease = $metadata.LTSRelease.Latest From 0b5614ff9098b43a45097bdc3d216187909f87f0 Mon Sep 17 00:00:00 2001 From: Patrick Meinecke Date: Mon, 23 Sep 2024 14:12:34 -0400 Subject: [PATCH 5/7] Prefix the correct parameter --- .pipelines/templates/release-upload-buildinfo.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pipelines/templates/release-upload-buildinfo.yml b/.pipelines/templates/release-upload-buildinfo.yml index d03de64da0d..e57cc0e0455 100644 --- a/.pipelines/templates/release-upload-buildinfo.yml +++ b/.pipelines/templates/release-upload-buildinfo.yml @@ -127,21 +127,21 @@ jobs: if ($env:CopyMainBuildInfo -eq 'YES') { $jsonFile = "$env:BuildInfoJsonFile" $blobName = Get-Item $jsonFile | Split-Path -Leaf - Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" - Set-AzStorageBlobContent -File "$prefix/$jsonFile" -Container $containerName -Blob $blobName -Context $storageContext -Force + Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$prefix/$blobName" + Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob "$prefix/$blobName" -Context $storageContext -Force } if ($env:CopyLTSBuildInfo -eq 'YES') { $jsonFile = "$env:LtsBuildInfoJsonFile" $blobName = Get-Item $jsonFile | Split-Path -Leaf - Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" - Set-AzStorageBlobContent -File "$prefix/$jsonFile" -Container $containerName -Blob $blobName -Context $storageContext -Force + Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$prefix/$blobName" + Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob "$prefix/$blobName" -Context $storageContext -Force } if ($env:CopyVersionBuildInfo -eq 'YES') { $jsonFile = "$env:VersionBuildInfoJsonFile" $blobName = Get-Item $jsonFile | Split-Path -Leaf - Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" - Set-AzStorageBlobContent -File "$prefix/$jsonFile" -Container $containerName -Blob $blobName -Context $storageContext -Force + Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$prefix/$blobName" + Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob "$prefix/$blobName" -Context $storageContext -Force } condition: and(succeeded(), eq(variables['CopyMainBuildInfo'], 'YES')) From 01fded1a119d8c82abceb25574fa94843135ab26 Mon Sep 17 00:00:00 2001 From: Patrick Meinecke Date: Tue, 24 Sep 2024 14:26:49 -0400 Subject: [PATCH 6/7] Add "install/" prefix in release artifacts --- .pipelines/templates/release-MakeBlobPublic.yml | 2 +- tools/install-powershell.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-MakeBlobPublic.yml b/.pipelines/templates/release-MakeBlobPublic.yml index c0104ac4c2a..e4f44c567e0 100644 --- a/.pipelines/templates/release-MakeBlobPublic.yml +++ b/.pipelines/templates/release-MakeBlobPublic.yml @@ -62,7 +62,7 @@ jobs: $sourceStorageAccountName = '$(StorageAccount)' $destinationStorageAccountName = '$(PSInfraStorageAccount)' $destinationContainerName = '$web' - $destinationPrefix = '$(ReleaseTagVar)' + $destinationPrefix = 'install/$(ReleaseTagVar)' $sourceContext = New-AzStorageContext -StorageAccountName $sourceStorageAccountName Write-Verbose -Verbose "Source context: $($sourceContext.BlobEndPoint)" diff --git a/tools/install-powershell.ps1 b/tools/install-powershell.ps1 index 3cd5fd464f4..414f0b35bb3 100644 --- a/tools/install-powershell.ps1 +++ b/tools/install-powershell.ps1 @@ -296,7 +296,7 @@ try { throw "The OS architecture is '$architecture'. However, we currently only support daily package for x64." } - $downloadURL = "https://powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net/$($metadata.ReleaseTag)/$packageName" + $downloadURL = "https://powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net/install/$($metadata.ReleaseTag)/$packageName" Write-Verbose "About to download package from '$downloadURL'" -Verbose $packagePath = Join-Path -Path $tempDir -ChildPath $packageName From 06fd265db882e2fd7409cff07d8813a29048ca89 Mon Sep 17 00:00:00 2001 From: Patrick Meinecke Date: Tue, 24 Sep 2024 14:45:10 -0400 Subject: [PATCH 7/7] Remove blob before copy if it already exists --- .pipelines/templates/release-MakeBlobPublic.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pipelines/templates/release-MakeBlobPublic.yml b/.pipelines/templates/release-MakeBlobPublic.yml index e4f44c567e0..559509cb5e2 100644 --- a/.pipelines/templates/release-MakeBlobPublic.yml +++ b/.pipelines/templates/release-MakeBlobPublic.yml @@ -84,6 +84,11 @@ jobs: $destinationBlobName = "$destinationPrefix/$sourceBlobName" Write-Verbose -Verbose "destinationBlobName = $destinationBlobName" + $existingBlob = Get-AzStorageBlob -Blob $destinationBlobName -Container $destinationContainerName -Context $destinationContext -ErrorAction Ignore + if ($existingBlob) { + Write-Verbose -Verbose "Blob $destinationBlobName already exists in '$destinationStorageAccountName/$destinationContainerName', removing before copy." + $existingBlob | Remove-AzStorageBlob -ErrorAction Stop -Verbose + } Copy-AzStorageBlob -SourceContext $sourceContext -DestinationContext $destinationContext -SrcContainer $sourceContainerName -SrcBlob $sourceBlobName -DestContainer $destinationContainerName -DestBlob $destinationBlobName -Force -Verbose -Confirm:$false }