From ad613cb436ed01df0b5e7e06cf47f282ac37346c Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 19 Mar 2019 16:35:02 -0700 Subject: [PATCH 1/4] Create NuGet packages from IL assemblies --- tools/packaging/packaging.psm1 | 64 +++---------------- .../azureDevOps/templates/nuget.yml | 51 +-------------- 2 files changed, 11 insertions(+), 104 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 846fe065cc5..ed6f36ec89f 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1444,7 +1444,7 @@ function CreateNugetPlatformFolder Creates NuGet packages containing linux, osx and Windows runtime assemblies. .DESCRIPTION -Creates a NuGet package for linux, osx, Windows runtimes for 32 bit, 64 bit and ARM. +Creates a NuGet package of IL assemblies for unix and windows. The packages for Microsoft.PowerShell.Commands.Diagnostics, Microsoft.PowerShell.Commands.Management, Microsoft.PowerShell.Commands.Utility, Microsoft.PowerShell.ConsoleHost, Microsoft.PowerShell.CoreCLR.Eventing, Microsoft.PowerShell.SDK, Microsoft.PowerShell.Security, Microsoft.WSMan.Management, Microsoft.WSMan.Runtime, @@ -1456,31 +1456,16 @@ Path where the package will be created. .PARAMETER PackageVersion Version of the created package. -.PARAMETER Winx86BinPath -Path to folder containing Windows x86 assemblies. +.PARAMETER WinFxdBinPath +Path to folder containing Windows framework dependent assemblies. -.PARAMETER Winx64BinPath -Path to folder containing Windows x64 assemblies. - -.PARAMETER WinArm32BinPath -Path to folder containing Windows arm32 assemblies. - -.PARAMETER WinArm64BinPath -Path to folder containing Windows arm64 assemblies. - -.PARAMETER LinuxArm32BinPath -Path to folder containing linux arm32 assemblies. - -.PARAMETER LinuxBinPath -Path to folder containing linux x64 assemblies. - -.PARAMETER OsxBinPath -Path to folder containing osx assemblies. +.PARAMETER LinuxFxdBinPath +Path to folder containing Linux framework dependent assemblies. .PARAMETER GenAPIToolPath Path to the GenAPI.exe tool. #> -function New-UnifiedNugetPackage +function New-ILNugetPackage { [CmdletBinding(SupportsShouldProcess = $true)] param( @@ -1492,28 +1477,10 @@ function New-UnifiedNugetPackage [string] $PackageVersion, [Parameter(Mandatory = $true)] - [string] $Winx86BinPath, - - [Parameter(Mandatory = $true)] - [string] $Winx64BinPath, - - [Parameter(Mandatory = $true)] - [string] $WinArm32BinPath, - - [Parameter(Mandatory = $true)] - [string] $WinArm64BinPath, - - [Parameter(Mandatory = $true)] - [string] $LinuxArm32BinPath, - - [Parameter(Mandatory = $false)] - [string] $LinuxAlpineBinPath, + [string] $WinFxdBinPath, [Parameter(Mandatory = $true)] - [string] $LinuxBinPath, - - [Parameter(Mandatory = $true)] - [string] $OsxBinPath, + [string] $LinuxFxdBinPath, [Parameter(Mandatory = $true)] [string] $GenAPIToolPath @@ -1565,22 +1532,11 @@ function New-UnifiedNugetPackage $packageRuntimesFolderPath = $packageRuntimesFolder.FullName - CreateNugetPlatformFolder -Platform 'win-x86' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $winX86BinPath - CreateNugetPlatformFolder -Platform 'win-x64' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $winX64BinPath - CreateNugetPlatformFolder -Platform 'win-arm' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $winArm32BinPath - CreateNugetPlatformFolder -Platform 'win-arm64' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $winArm64BinPath + CreateNugetPlatformFolder -Platform 'win' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $WinFxdBinPath if ($linuxExceptionList -notcontains $file ) { - CreateNugetPlatformFolder -Platform 'linux-arm' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $linuxArm32BinPath - - if ($linuxAlpineBinPath) - { - CreateNugetPlatformFolder -Platform 'alpine-x64' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $LinuxAlpineBinPath - } - - CreateNugetPlatformFolder -Platform 'linux-x64' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $linuxBinPath - CreateNugetPlatformFolder -Platform 'osx' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $osxBinPath + CreateNugetPlatformFolder -Platform 'unix' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $LinuxFxdBinPath } #region nuspec diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index 33e985e5c3d..32f41e5da4b 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -9,13 +9,6 @@ jobs: condition: succeeded() pool: PowerShell variables: - linuxArm32Path: '$(System.ArtifactsDirectory)/linuxArm32' - winArm32Path: '$(System.ArtifactsDirectory)/winArm32' - linuxX64Path: '$(System.ArtifactsDirectory)/linuxX64' - macOSPath: '$(System.ArtifactsDirectory)/macOS' - winArm64Path: '$(System.ArtifactsDirectory)/winArm64' - winX64Path: '$(System.ArtifactsDirectory)/winX64' - winX86Path: '$(System.ArtifactsDirectory)/winX86' GenAPIToolPath: '$(System.ArtifactsDirectory)/GenAPI' PackagePath: '$(System.ArtifactsDirectory)/UnifiedPackagePath' winFxdPath: '$(System.ArtifactsDirectory)/winFxd' @@ -54,48 +47,6 @@ jobs: Get-ChildItem $packagePath -Recurse displayName: 'Conflate packages to same folder' - - task: ExtractFiles@1 - displayName: 'Extract files linuxArm32' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/powershell-*-linux-arm32.tar.gz' - destinationFolder: '$(linuxArm32Path)' - - - task: ExtractFiles@1 - displayName: 'Extract files linuxX64' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/powershell-*-linux-x64.tar.gz' - destinationFolder: '$(linuxX64Path)' - - - task: ExtractFiles@1 - displayName: 'Extract files macOS files' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/powershell-*-osx*.tar.gz' - destinationFolder: '$(macOSPath)' - - - task: ExtractFiles@1 - displayName: 'Extract files win-arm32' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/PowerShell-*-win-arm32.zip' - destinationFolder: '$(winArm32Path)' - - - task: ExtractFiles@1 - displayName: 'Extract files win-arm64' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/PowerShell-*-win-arm64.zip' - destinationFolder: '$(winArm64Path)' - - - task: ExtractFiles@1 - displayName: 'Extract files win-X64' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/PowerShell-*-win-x64.zip' - destinationFolder: '$(winX64Path)' - - - task: ExtractFiles@1 - displayName: 'Extract files win-X86' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/PowerShell-*-win-x86.zip' - destinationFolder: '$(winX86Path)' - - task: ExtractFiles@1 displayName: 'Extract files win-fxdependent' inputs: @@ -128,7 +79,7 @@ jobs: - powershell: | Import-Module $env:BUILD_SOURCESDIRECTORY\build.psm1 Import-Module $env:BUILD_SOURCESDIRECTORY\tools\packaging - New-UnifiedNugetPackage -PackagePath "$(PackagePath)" -PackageVersion "$(Version)" -winx86BinPath "$(winX86Path)" -winx64BinPath "$(winX64Path)" -winArm32BinPath "$(winArm32Path)" -winArm64BinPath "$(winArm64Path)" -linuxArm32BinPath "$(linuxArm32Path)" -linuxBinPath "$(linuxX64Path)" -osxBinPath "$(macOSPath)" -GenAPIToolPath "$(GenAPIToolPath)" + New-ILNugetPackage -PackagePath "$(PackagePath)" -PackageVersion "$(Version)" -WinBinPath '$(winFxdPath)' -LinuxBinPath '$(linuxFxdPath)' -GenAPIToolPath "$(GenAPIToolPath)" displayName: 'Create Nuget Package Folders' - powershell: | From e3b5b5a1259618cb1d654c8de71cae2d512d3234 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 19 Mar 2019 17:25:00 -0700 Subject: [PATCH 2/4] Add Replacement patterns --- tools/packaging/packaging.psd1 | 2 +- tools/packaging/packaging.psm1 | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tools/packaging/packaging.psd1 b/tools/packaging/packaging.psd1 index d474f799727..f9e98221ce2 100644 --- a/tools/packaging/packaging.psd1 +++ b/tools/packaging/packaging.psd1 @@ -6,7 +6,7 @@ Copyright="Copyright (c) Microsoft Corporation. All rights reserved." ModuleVersion="1.0.0" PowerShellVersion="5.0" CmdletsToExport=@() -FunctionsToExport=@('Start-PSPackage','New-PSSignedBuildZip', 'New-UnifiedNugetPackage', 'New-MSIPatch', 'Expand-PSSignedBuild', 'Publish-NugetToMyGet', 'New-DotnetSdkContainerFxdPackage', 'New-GlobalToolNupkg') +FunctionsToExport=@('Start-PSPackage','New-PSSignedBuildZip', 'New-UnifiedNugetPackage', 'New-MSIPatch', 'Expand-PSSignedBuild', 'Publish-NugetToMyGet', 'New-DotnetSdkContainerFxdPackage', 'New-GlobalToolNupkg', 'New-ILNugetPackage') RootModule="packaging.psm1" RequiredModules = @("build") } diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index ed6f36ec89f..95d4f7f268d 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1515,7 +1515,7 @@ function New-ILNugetPackage $refBinPath = New-TempFolder $SnkFilePath = "$RepoRoot\src\signing\visualstudiopublic.snk" - New-ReferenceAssembly -linux64BinPath $linuxBinPath -RefAssemblyDestinationPath $refBinPath -RefAssemblyVersion $PackageVersion -SnkFilePath $SnkFilePath -GenAPIToolPath $GenAPIToolPath + New-ReferenceAssembly -linux64BinPath $LinuxFxdBinPath -RefAssemblyDestinationPath $refBinPath -RefAssemblyVersion $PackageVersion -SnkFilePath $SnkFilePath -GenAPIToolPath $GenAPIToolPath foreach ($file in $fileList) { @@ -1859,7 +1859,7 @@ function New-ReferenceAssembly throw "$assemblyName.dll was not found at: $Linux64BinPath" } - $genAPIArgs = "$linuxDllPath","-libPath:$Linux64BinPath" + $genAPIArgs = "$linuxDllPath","-libPath:$Linux64BinPath,$Linux64BinPath\ref" Write-Log "GenAPI cmd: $genAPIExe $genAPIArgsString" Start-NativeExecution { & $genAPIExe $genAPIArgs } | Out-File $generatedSource -Force @@ -1951,6 +1951,26 @@ function CleanupGeneratedSourceCode ApplyTo = "Microsoft.PowerShell.Commands.Utility" Pattern = "public partial struct ConvertToJsonContext" Replacement = "public readonly struct ConvertToJsonContext" + }, + @{ + ApplyTo = "Microsoft.PowerShell.Commands.Utility" + Pattern = "Unable to resolve assembly 'Assembly(Name=Newtonsoft.Json" + Replacement = "// Unable to resolve assembly 'Assembly(Name=Newtonsoft.Json" + }, + @{ + ApplyTo = "System.Management.Automation" + Pattern = "Unable to resolve assembly 'Assembly(Name=System.Security.Principal.Windows" + Replacement = "// Unable to resolve assembly 'Assembly(Name=System.Security.Principal.Windows" + }, + @{ + ApplyTo = "System.Management.Automation" + Pattern = "Unable to resolve assembly 'Assembly(Name=Microsoft.Management.Infrastructure" + Replacement = "// Unable to resolve assembly 'Assembly(Name=Microsoft.Management.Infrastructure" + }, + @{ + ApplyTo = "System.Management.Automation" + Pattern = "Unable to resolve assembly 'Assembly(Name=System.Security.AccessControl" + Replacement = "// Unable to resolve assembly 'Assembly(Name=System.Security.AccessControl" } ) From 42bb52f23b9c7922951f079689bda5d8ff362ece Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 19 Mar 2019 19:10:28 -0700 Subject: [PATCH 3/4] Remove references to New-UnifiedNugetPackage --- test/hosting/NuGet.Config | 1 + tools/packaging/packaging.psd1 | 2 +- tools/packaging/packaging.psm1 | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/hosting/NuGet.Config b/test/hosting/NuGet.Config index c2c6e1b7c2d..2234c20c3be 100644 --- a/test/hosting/NuGet.Config +++ b/test/hosting/NuGet.Config @@ -2,6 +2,7 @@ + diff --git a/tools/packaging/packaging.psd1 b/tools/packaging/packaging.psd1 index f9e98221ce2..de5672690e2 100644 --- a/tools/packaging/packaging.psd1 +++ b/tools/packaging/packaging.psd1 @@ -6,7 +6,7 @@ Copyright="Copyright (c) Microsoft Corporation. All rights reserved." ModuleVersion="1.0.0" PowerShellVersion="5.0" CmdletsToExport=@() -FunctionsToExport=@('Start-PSPackage','New-PSSignedBuildZip', 'New-UnifiedNugetPackage', 'New-MSIPatch', 'Expand-PSSignedBuild', 'Publish-NugetToMyGet', 'New-DotnetSdkContainerFxdPackage', 'New-GlobalToolNupkg', 'New-ILNugetPackage') +FunctionsToExport=@('Start-PSPackage','New-PSSignedBuildZip', 'New-MSIPatch', 'Expand-PSSignedBuild', 'Publish-NugetToMyGet', 'New-DotnetSdkContainerFxdPackage', 'New-GlobalToolNupkg', 'New-ILNugetPackage') RootModule="packaging.psm1" RequiredModules = @("build") } diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 95d4f7f268d..23250a7db01 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1488,7 +1488,7 @@ function New-ILNugetPackage if (-not $Environment.IsWindows) { - throw "New-UnifiedNugetPackage can be only executed on Windows platform." + throw "New-ILNugetPackage can be only executed on Windows platform." } $fileList = @( @@ -1647,7 +1647,7 @@ function New-ILNugetPackage <# Copy the generated reference assemblies to the 'ref/netcoreapp2.1' folder properly. - This is a helper function used by 'New-UnifiedNugetPackage' + This is a helper function used by 'New-ILNugetPackage' #> function CopyReferenceAssemblies { From f6503bece4a1708a55c0c97bdd95c96f346d908e Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 20 Mar 2019 09:34:38 -0700 Subject: [PATCH 4/4] Revert test change and fix parameter names in build --- test/hosting/NuGet.Config | 1 - tools/releaseBuild/azureDevOps/templates/nuget.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/test/hosting/NuGet.Config b/test/hosting/NuGet.Config index 2234c20c3be..c2c6e1b7c2d 100644 --- a/test/hosting/NuGet.Config +++ b/test/hosting/NuGet.Config @@ -2,7 +2,6 @@ - diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index 32f41e5da4b..91273416864 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -79,7 +79,7 @@ jobs: - powershell: | Import-Module $env:BUILD_SOURCESDIRECTORY\build.psm1 Import-Module $env:BUILD_SOURCESDIRECTORY\tools\packaging - New-ILNugetPackage -PackagePath "$(PackagePath)" -PackageVersion "$(Version)" -WinBinPath '$(winFxdPath)' -LinuxBinPath '$(linuxFxdPath)' -GenAPIToolPath "$(GenAPIToolPath)" + New-ILNugetPackage -PackagePath "$(PackagePath)" -PackageVersion "$(Version)" -WinFxdBinPath '$(winFxdPath)' -LinuxFxdBinPath '$(linuxFxdPath)' -GenAPIToolPath "$(GenAPIToolPath)" displayName: 'Create Nuget Package Folders' - powershell: |