diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 11590184531a..996bb78d590a 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -26,6 +26,8 @@ variables: - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: _DotNetPublishToBlobFeed value: true + - name: Codeql.Enabled + value: true - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal @@ -101,11 +103,6 @@ stages: _PublishArgs: '' _SignType: test _Test: -test - Build_Release: - _BuildConfig: Release - _PublishArgs: '' - _SignType: test - _Test: -test - template: /eng/build.yml parameters: diff --git a/NuGet.config b/NuGet.config index 32a56d033347..46a503589894 100644 --- a/NuGet.config +++ b/NuGet.config @@ -4,20 +4,21 @@ - + + - + - + - + @@ -37,16 +38,17 @@ - + - + + - + - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index db60bce73ccc..88addcf9979f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,70 +1,70 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-templating - 84fa0403a62bd95b9e1f29c00f4b4f8e605bfc7c + 3ef5e36b6022edc32ce62b1b85e964e0fefb0deb - + https://dev.azure.com/dnceng/internal/_git/dotnet-templating - 84fa0403a62bd95b9e1f29c00f4b4f8e605bfc7c + 3ef5e36b6022edc32ce62b1b85e964e0fefb0deb - + https://dev.azure.com/dnceng/internal/_git/dotnet-templating - 84fa0403a62bd95b9e1f29c00f4b4f8e605bfc7c + 3ef5e36b6022edc32ce62b1b85e964e0fefb0deb - + https://dev.azure.com/dnceng/internal/_git/dotnet-templating - 84fa0403a62bd95b9e1f29c00f4b4f8e605bfc7c + 3ef5e36b6022edc32ce62b1b85e964e0fefb0deb - + https://dev.azure.com/dnceng/internal/_git/dotnet-templating - 84fa0403a62bd95b9e1f29c00f4b4f8e605bfc7c + 3ef5e36b6022edc32ce62b1b85e964e0fefb0deb - + https://dev.azure.com/dnceng/internal/_git/dotnet-templating - 84fa0403a62bd95b9e1f29c00f4b4f8e605bfc7c + 3ef5e36b6022edc32ce62b1b85e964e0fefb0deb - + https://dev.azure.com/dnceng/internal/_git/dotnet-templating - 84fa0403a62bd95b9e1f29c00f4b4f8e605bfc7c + 3ef5e36b6022edc32ce62b1b85e964e0fefb0deb - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 943474ca16db7c65ba6cff4a89c3ebd219dde3e5 + 02e45a41b756dd147fc39297df7649d86fa245e7 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 943474ca16db7c65ba6cff4a89c3ebd219dde3e5 + 02e45a41b756dd147fc39297df7649d86fa245e7 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 943474ca16db7c65ba6cff4a89c3ebd219dde3e5 + 02e45a41b756dd147fc39297df7649d86fa245e7 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 943474ca16db7c65ba6cff4a89c3ebd219dde3e5 + 02e45a41b756dd147fc39297df7649d86fa245e7 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 943474ca16db7c65ba6cff4a89c3ebd219dde3e5 + 02e45a41b756dd147fc39297df7649d86fa245e7 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 943474ca16db7c65ba6cff4a89c3ebd219dde3e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 943474ca16db7c65ba6cff4a89c3ebd219dde3e5 + 02e45a41b756dd147fc39297df7649d86fa245e7 https://github.com/dotnet/runtime 4822e3c3aa77eb82b2fb33c9321f923cf11ddde6 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 943474ca16db7c65ba6cff4a89c3ebd219dde3e5 + 02e45a41b756dd147fc39297df7649d86fa245e7 https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted @@ -113,13 +113,13 @@ https://github.com/dotnet/roslyn 432d17a83aa5623e4581756f4d642997048056bd - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted @@ -166,66 +166,66 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 70ae3df4a6f3c92fb6b315afc405edd10ff38579 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - ee26d5bddd7cf13b37987de68fee12d9e50ff33d + e8b4ef2cfc595ccca430b7a3c31cad2855e003eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - ee26d5bddd7cf13b37987de68fee12d9e50ff33d + e8b4ef2cfc595ccca430b7a3c31cad2855e003eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - ee26d5bddd7cf13b37987de68fee12d9e50ff33d + e8b4ef2cfc595ccca430b7a3c31cad2855e003eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - ee26d5bddd7cf13b37987de68fee12d9e50ff33d + e8b4ef2cfc595ccca430b7a3c31cad2855e003eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - 96c5254e5ca5040021caf779216b1b3641875b1e + 4178d795849932702f98b7f375f6341267510326 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 https://github.com/dotnet/razor-compiler @@ -242,21 +242,21 @@ fde127ba1cbee8a15c4a1fb8e46e686f3caac8ff - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 617d594f2bf75a8904d3d0e7d68a0bacf8e6763a + 54142bcb25544b72fac28b1b061e7788c91c4f49 https://github.com/aspnet/xdt @@ -287,22 +287,22 @@ - + https://github.com/dotnet/arcade - 1a6b24397e50146d0fece9cfb9c0b87275691e6f + 3a5fc99d467769acc6f951cfefef5a9718ecfc42 - + https://github.com/dotnet/arcade - 1a6b24397e50146d0fece9cfb9c0b87275691e6f + 3a5fc99d467769acc6f951cfefef5a9718ecfc42 - + https://github.com/dotnet/arcade - 1a6b24397e50146d0fece9cfb9c0b87275691e6f + 3a5fc99d467769acc6f951cfefef5a9718ecfc42 - + https://github.com/dotnet/arcade - 1a6b24397e50146d0fece9cfb9c0b87275691e6f + 3a5fc99d467769acc6f951cfefef5a9718ecfc42 https://github.com/dotnet/runtime diff --git a/eng/Versions.props b/eng/Versions.props index 8365bc5eeb51..6dbdeddba08a 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -10,7 +10,7 @@ - 6.0.306 + 6.0.307 servicing true @@ -28,7 +28,7 @@ $(NewtonsoftJsonPackageVersion) 4.0.0 5.0.0 - 6.0.0-beta.22212.5 + 6.0.0-beta.22564.9 3.1.0 6.0.0 4.3.0 @@ -43,13 +43,13 @@ - 6.0.11 - 6.0.11-servicing.22523.4 - 6.0.11 + 6.0.12 + 6.0.12-servicing.22568.7 + 6.0.12 $(MicrosoftNETCoreAppRuntimewinx64PackageVersion) 6.0.0 - 6.0.11 - 6.0.11-servicing.22523.4 + 6.0.12 + 6.0.12-servicing.22568.7 6.0.0-preview.7.21363.9 6.0.0 @@ -112,11 +112,11 @@ - 6.0.306-servicing.22523.5 - 6.0.306 - 6.0.306 - 6.0.306 - 6.0.306 + 6.0.307-servicing.22575.5 + 6.0.307 + 6.0.307 + 6.0.307 + 6.0.307 @@ -133,23 +133,23 @@ - 6.0.11-servicing.22523.4 - 6.0.11 - 6.0.11-servicing.22523.4 - 6.0.11-servicing.22523.4 - 6.0.11-servicing.22523.4 - 6.0.11 + 6.0.12-servicing.22571.9 + 6.0.12 + 6.0.12-servicing.22571.9 + 6.0.12-servicing.22571.9 + 6.0.12-servicing.22571.9 + 6.0.12 6.0.3-1.22213.1 6.0.3-1.22213.1 6.0.3-1.22213.1 - 6.0.11-servicing.22523.2 + 6.0.12-servicing.22569.2 - 6.0.11-servicing.22523.3 + 6.0.12-servicing.22578.4 @@ -168,7 +168,7 @@ 4.19.2 4.19.0 - 6.0.0-beta.22212.5 + 6.0.0-beta.22564.9 4.8.2 6.0.0-beta.22262.1 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 8943da242f6e..33a6f2d0e248 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -26,6 +26,7 @@ Param( [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '', [switch] $excludePrereleaseVS, + [switch] $nativeToolsOnMachine, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) @@ -67,6 +68,7 @@ function Print-Usage() { Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" + Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." @@ -146,6 +148,9 @@ try { $nodeReuse = $false } + if ($nativeToolsOnMachine) { + $env:NativeToolsOnMachine = $true + } if ($restore) { InitializeNativeTools } diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index fdfeea66e7d4..b09ea669f9c0 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -52,7 +52,7 @@ done # Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples cpuname=$(uname -m) case $cpuname in - aarch64) + arm64|aarch64) buildarch=arm64 ;; amd64|x86_64) @@ -61,7 +61,7 @@ case $cpuname in armv*l) buildarch=arm ;; - i686) + i[3-6]86) buildarch=x86 ;; *) diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1 index a733a8885824..3e5c1c74a1c5 100644 --- a/eng/common/generate-sbom-prep.ps1 +++ b/eng/common/generate-sbom-prep.ps1 @@ -2,6 +2,8 @@ Param( [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed ) +. $PSScriptRoot\pipeline-logging-functions.ps1 + Write-Host "Creating dir $ManifestDirPath" # create directory for sbom manifest to be placed if (!(Test-Path -path $ManifestDirPath)) diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh index f6c77453142a..d5c76dc827b4 100644 --- a/eng/common/generate-sbom-prep.sh +++ b/eng/common/generate-sbom-prep.sh @@ -2,6 +2,18 @@ source="${BASH_SOURCE[0]}" +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +. $scriptroot/pipeline-logging-functions.sh + manifest_dir=$1 if [ ! -d "$manifest_dir" ] ; then diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index db830c00a6f8..6c7a851a8087 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -31,6 +31,10 @@ Wait time between retry attempts in seconds .PARAMETER GlobalJsonFile File path to global.json file +.PARAMETER PathPromotion +Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) +or break the build if a native tool is not found on the path (on a local dev machine) + .NOTES #> [CmdletBinding(PositionalBinding=$false)] @@ -41,7 +45,8 @@ Param ( [switch] $Force = $False, [int] $DownloadRetries = 5, [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile + [string] $GlobalJsonFile, + [switch] $PathPromotion ) if (!$GlobalJsonFile) { @@ -77,53 +82,101 @@ try { ConvertFrom-Json | Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue if ($NativeTools) { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } - - if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } - } - if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { - if($Force) { - $LocalInstallerArguments += @{ Force = $True } - } - } - if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } - } - - Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" - & $InstallerPath @LocalInstallerArguments - if ($LASTEXITCODE -Ne "0") { - $errMsg = "$ToolName installation failed" - if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { - $showNativeToolsWarning = $true - if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { - $showNativeToolsWarning = $false + if ($PathPromotion -eq $True) { + if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $InstalledTools = @{} + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + if ($ToolVersion -eq "latest") { + $ToolVersion = "" + } + $ArcadeToolsDirectory = "C:\arcade-tools" + if (-not (Test-Path $ArcadeToolsDirectory)) { + Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed." + exit 1 + } + $ToolDirectory = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending)[0] + if ([string]::IsNullOrWhiteSpace($ToolDirectory)) { + Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." + exit 1 } - if ($showNativeToolsWarning) { - Write-Warning $errMsg + $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" + if (-not (Test-Path -Path "$BinPathFile")) { + Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." + exit 1 } - $toolInstallationFailure = $true - } else { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host $errMsg - exit 1 + $BinPath = Get-Content "$BinPathFile" + $ToolPath = Convert-Path -Path $BinPath + Write-Host "Adding $ToolName to the path ($ToolPath)..." + Write-Host "##vso[task.prependpath]$ToolPath" + $env:PATH = "$ToolPath;$env:PATH" + $InstalledTools += @{ $ToolName = $ToolDirectory.FullName } + } } + return $InstalledTools + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + } + } + exit 0 + } + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } + + if ($Verbose) { + $LocalInstallerArguments += @{ Verbose = $True } + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerArguments += @{ Force = $True } + } + } + if ($Clean) { + $LocalInstallerArguments += @{ Clean = $True } + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" + & $InstallerPath @LocalInstallerArguments + if ($LASTEXITCODE -Ne "0") { + $errMsg = "$ToolName installation failed" + if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { + $showNativeToolsWarning = $true + if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { + $showNativeToolsWarning = $false + } + if ($showNativeToolsWarning) { + Write-Warning $errMsg + } + $toolInstallationFailure = $true + } else { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host $errMsg + exit 1 + } + } + } + + if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host 'Native tools bootstrap failed' + exit 1 } - } - - if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host 'Native tools bootstrap failed' - exit 1 } } else { @@ -139,7 +192,7 @@ try { Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" return $InstallBin } - else { + elseif (-not ($PathPromotion)) { Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' exit 1 } @@ -149,4 +202,4 @@ catch { Write-Host $_.ScriptStackTrace Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_ ExitWithExitCode 1 -} +} \ No newline at end of file diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config new file mode 100644 index 000000000000..19d3d311b166 --- /dev/null +++ b/eng/common/internal/NuGet.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index beb9c4648ea1..7f5ce6d60813 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -8,6 +8,9 @@ + + + diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1 new file mode 100644 index 000000000000..648c5068d7d6 --- /dev/null +++ b/eng/common/sdl/sdl.ps1 @@ -0,0 +1,38 @@ + +function Install-Gdn { + param( + [Parameter(Mandatory=$true)] + [string]$Path, + + # If omitted, install the latest version of Guardian, otherwise install that specific version. + [string]$Version + ) + + $ErrorActionPreference = 'Stop' + Set-StrictMode -Version 2.0 + $disableConfigureToolsetImport = $true + $global:LASTEXITCODE = 0 + + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") + + if ($Version) { + $argumentList += "-Version $Version" + } + + Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait + + $gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path + + if (!$gdnCliPath) + { + Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian' + } + + return $gdnCliPath.FullName +} \ No newline at end of file diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 24cec0424e5d..aaeb83b4dcbd 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -53,8 +53,8 @@ jobs: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals windows.vs2019.amd64 steps: - checkout: self clean: true diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 9d1e3042d8a6..5b1b77d1c74a 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -40,8 +40,8 @@ jobs: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals windows.vs2019.amd64 variables: - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 88f6f75a622d..b6137f44ada1 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -46,10 +46,10 @@ jobs: # source-build builds run in Docker, including the default managed platform. pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Public + name: NetCore-Svc-Public demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Internal + name: NetCore1ESPool-Svc-Internal demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 554e71cfc436..9c4becbc8592 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -89,8 +89,8 @@ jobs: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals windows.vs2019.amd64 runAsPublic: ${{ parameters.runAsPublic }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 24e92a275a4f..fc022ca9b266 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -100,8 +100,8 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml @@ -137,8 +137,8 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml parameters: @@ -197,8 +197,8 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml parameters: @@ -254,8 +254,8 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml parameters: diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml index 7b8ee18a28d7..9dd5709f66dc 100644 --- a/eng/common/templates/steps/execute-sdl.yml +++ b/eng/common/templates/steps/execute-sdl.yml @@ -8,29 +8,28 @@ parameters: condition: '' steps: -- ${{ if ne(parameters.overrideGuardianVersion, '') }}: - - powershell: | - $content = Get-Content $(GuardianPackagesConfigFile) - - Write-Host "packages.config content was:`n$content" - - $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)') - $content | Set-Content $(GuardianPackagesConfigFile) - - Write-Host "packages.config content updated to:`n$content" - displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }} +- task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: GuardianConnect - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' -- task: NuGetCommand@2 - displayName: 'Install Guardian' - inputs: - restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - feedsToUse: config - nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config - externalFeedCredentials: GuardianConnect - restoreDirectory: $(Build.SourcesDirectory)\.packages +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian (Overridden) + +- ${{ if eq(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian - ${{ if ne(parameters.overrideParameters, '') }}: - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} @@ -40,7 +39,7 @@ steps: - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: ${{ parameters.executeAllSdlToolsScript }} - -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) + -GuardianCliLocation $(GuardianCliLocation) -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} @@ -62,7 +61,28 @@ steps: c i condition: succeededOrFailed() + - publish: $(Agent.BuildDirectory)/.gdn artifact: GuardianConfiguration displayName: Publish GuardianConfiguration + condition: succeededOrFailed() + + # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration + # with the "SARIF SAST Scans Tab" Azure DevOps extension + - task: CopyFiles@2 + displayName: Copy SARIF files + inputs: + flattenFolders: true + sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ + contents: '**/*.sarif' + targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs + condition: succeededOrFailed() + + # Use PublishBuildArtifacts because the SARIF extension only checks this case + # see microsoft/sarif-azuredevops-extension#4 + - task: PublishBuildArtifacts@1 + displayName: Publish SARIF files to CodeAnalysisLogs container + inputs: + pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs + artifactName: CodeAnalysisLogs condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index abb1b2bcda42..b5b3e5aeb3b9 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -63,6 +63,11 @@ steps: targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' fi + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + publishArgs= if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then publishArgs='--publish' @@ -75,6 +80,7 @@ steps: $internalRuntimeDownloadArgs \ $internalRestoreArgs \ $targetRidArgs \ + $runtimeOsArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true displayName: Build diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index f1e1cb53953b..81d7b0355e38 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -635,6 +635,10 @@ function InitializeNativeTools() { InstallDirectory = "$ToolsDir" } } + if ($env:NativeToolsOnMachine) { + Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..." + $nativeArgs += @{ PathPromotion = $true } + } & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs } } diff --git a/global.json b/global.json index e66fba4efbc4..6e4ab3442d46 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.22212.5", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.22212.5" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.22564.9", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.22564.9" } }