8000 Source-build symbols repackaging (#17454) · dotnet/sdk@447d4a3 · GitHub
[go: up one dir, main page]

Skip to content

Commit 447d4a3

Browse files
Source-build symbols repackaging (#17454)
Co-authored-by: Michael Simons <msimons@microsoft.com>
1 parent 0c19594 commit 447d4a3

File tree

2 files changed

+235
-1
lines changed

2 files changed

+235
-1
lines changed

src/SourceBuild/content/build.proj

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
<UsingTask AssemblyFile="$(LeakDetectionTasksAssembly)" TaskName="CheckForPoison" />
55
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WriteUsageBurndownData" />
66
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="ReplaceTextInFile" />
7+
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="CreateSdkSymbolsLayout" />
78

89
<Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
910

@@ -57,6 +58,84 @@
5758
<MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
5859
</Target>
5960

61+
<Target Name="DiscoverSymbolsTarballs"
62+
AfterTargets="Build">
63+
<ItemGroup>
64+
<SymbolsTarball Include="$(OutputPath)Symbols.*.tar.gz" />
65+
</ItemGroup>
66+
</Target>
67+
68+
<Target Name="ExtractSymbolsTarballs"
69+
AfterTargets="Build"
70+
DependsOnTargets="DiscoverSymbolsTarballs"
71+
Outputs="%(SymbolsTarball.Identity)">
72+
73+
<PropertyGroup>
74+
<Filename>$([System.IO.Path]::GetFileName('%(SymbolsTarball.Identity)'))</Filename>
75+
<RepositoryName>$(Filename.Split('.')[1])</RepositoryName>
76+
<UnifiedSymbolsLayout>$(ArtifactsTmpDir)Symbols</UnifiedSymbolsLayout>
77+
<DestinationFolder>$(UnifiedSymbolsLayout)/$(RepositoryName)</DestinationFolder>
78+
</PropertyGroup>
79+
80+
<MakeDir Directories="$(DestinationFolder)" />
81+
<Exec Command="tar -xzf %(SymbolsTarball.Identity) -C $(DestinationFolder)"
82+
WorkingDirectory="$(SymbolsRoot)" />
83+
84+
<Delete Files="%(SymbolsTarball.Identity)" />
85+
</Target>
86+
87+
<!-- After building, repackage symbols into a single tarball. -->
88+
<Target Name="RepackageSymbols"
89+
AfterTargets="Build"
90+
DependsOnTargets="
91+
DetermineMicrosoftSourceBuildIntermediateInstallerVersion;
92+
DiscoverSymbolsTarballs;
93+
ExtractSymbolsTarballs">
94+
<PropertyGroup>
95+
<UnifiedSymbolsTarball>$(OutputPath)dotnet-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</UnifiedSymbolsTarball>
96+
</PropertyGroup>
97+
98+
<Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
99+
WorkingDirectory="$(UnifiedSymbolsLayout)" />
100+
101+
<Message Importance="High" Text="Packaged all symbols in '$(UnifiedSymbolsTarball)'" />
102+
</Target>
103+
104+
<!-- After building, create the sdk symbols tarball. -->
105+
<Target Name="CreateSdkSymbolsTarball"
106+
AfterTargets="Build"
107+
DependsOnTargets="RepackageSymbols">
108+
<ItemGroup>
109+
<SdkTarballItem Include="$(OutputPath)dotnet-sdk-*$(TarBallExtension)"
110+
Exclude="$(OutputPath)dotnet-sdk-symbols-*$(TarBallExtension)" />
111+
</ItemGroup>
112+
113+
<PropertyGroup>
114+
<SdkSymbolsLayout>$(ArtifactsTmpDir)SdkSymbols</SdkSymbolsLayout>
115+
<SdkSymbolsTarball>$(OutputPath)dotnet-sdk-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</SdkSymbolsTarball>
116+
<SdkLayout>$(ArtifactsTmpDir)Sdk</SdkLayout>
117+
<SdkTarball>%(SdkTarballItem.Identity)</SdkTarball>
118+
</PropertyGroup>
119+
120+
<MakeDir Directories="$(SdkLayout)" />
121+
<Exec Command="tar -xzf $(SdkTarball) -C $(SdkLayout)"
122+
WorkingDirectory="$(OutputPath)" />
123+
124+
<CreateSdkSymbolsLayout SdkLayoutPath="$(SdkLayout)"
125+
AllSymbolsPath="$(UnifiedSymbolsLayout)"
126+
SdkSymbolsLayoutPath="$(SdkSymbolsLayout)"
127+
FailOnMissingPDBs="true" />
128+
129+
<Exec Command="tar --numeric-owner -czf $(SdkSymbolsTarball) *"
130+
WorkingDirectory="$(SdkSymbolsLayout)" />
131+
132+
<Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
133+
134+
<RemoveDir Directories="$(UnifiedSymbolsLayout)" />
135+
<RemoveDir Directories="$(SdkSymbolsLayout)" />
136+
<RemoveDir Directories="$(SdkLayout)" />
137+
</Target>
138+
60139
<!--
61140
Dev scenario: rewrite a prebuilt-report. This makes it easy to add data to an existing
62141
prebuilt report without performing another full build. This doesn't reevalutate which packages
@@ -104,7 +183,8 @@
104183

105184
<Target Name="RunSmokeTest">
106185
<ItemGroup>
107-
<SdkTarballItem Include="$(SourceBuiltTarBallPath)**/dotnet-sdk*$(TarBallExtension)" />
186+
<SdkTarballItem Include="$(SourceBuiltTarBallPath)**/dotnet-sdk*$(TarBallExtension)"
187+
Exclude="$(SourceBuiltTarBallPath)**/dotnet-sdk-symbols*$(TarBallExtension)" />
108188
<SourceBuiltArtifactsItem Include="$(SourceBuiltTarBallPath)**/Private.SourceBuilt.Artifacts.*$(TarBallExtension)" />
109189
</ItemGroup>
110190

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
using System.Collections;
7+
using System.Collections.Generic;
8+
using System.IO;
9+
using System.Linq;
10+
using System.Reflection.Metadata;
11+
using System.Reflection.PortableExecutable;
12+
using Microsoft.Build.Framework;
13+
using Microsoft.Build.Utilities;
14+
15+
namespace Microsoft.DotNet.Build.Tasks
16+
{
17+
// Creates a symbols layout that matches the SDK layout
18+
public class CreateSdkSymbolsLayout : Task
19+
{
20+
/// <summary>
21+
/// Path to SDK layout.
22+
/// </summary>
23+
[Required]
24+
public string SdkLayoutPath { get; set; }
25+
26+
/// <summary>
27+
/// Path to all source-built symbols, flat or with folder hierarchy.
28+
/// </summary>
29+
[Required]
30+
public string AllSymbolsPath { get; set; }
31+
32+
/// <summary>
33+
/// Path to SDK symbols layout - will be created if it doesn't exist.
34+
/// </summary>
35+
[Required]
36+
public string SdkSymbolsLayoutPath { get; set; }
37+
38+
/// <summary>
39+
/// If true, fails the build if any PDBs are missing.
40+
/// </summary>
41+
public bool FailOnMissingPDBs { get; set; }
42+
43+
public override bool Execute()
44+
{
45+
IList<string> filesWithoutPDBs = GenerateSymbolsLayout(IndexAllSymbols());
46+
if (filesWithoutPDBs.Count > 0)
47+
{
48+
LogErrorOrWarning(FailOnMissingPDBs, $"Did not find PDBs for the following SDK files:");
49+
foreach (string file in filesWithoutPDBs)
50+
{
51+
LogErrorOrWarning(FailOnMissingPDBs, file);
52+
}
53+
}
54+
55+
return !Log.HasLoggedErrors;
56+
}
57+
58+
private void LogErrorOrWarning(bool isError, string message)
59+
{
60+
if (FailOnMissingPDBs)
61+
{
62+
Log.LogError(message);
63+
}
64+
else
65+
{
66+
Log.LogWarning(message);
67+
}
68+
}
69+
70+
private IList<string> GenerateSymbolsLayout(Hashtable allPdbGuids)
71+
{
72+
List<string> filesWithoutPDBs = new List<string>();
73+
74+
if (Directory.Exists(SdkSymbolsLayoutPath))
75+
{
76+
Directory.Delete(SdkSymbolsLayoutPath, true);
77+
}
78+
79+
foreach (string file in Directory.GetFiles(SdkLayoutPath, "*", SearchOption.AllDirectories))
80+
{
81+
if (file.EndsWith(".dll", StringComparison.InvariantCultureIgnoreCase) &&
82+
!file.EndsWith(".resources.dll", StringComparison.InvariantCultureIgnoreCase))
83+
{
84+
string guid = string.Empty;
85+
using var pdbStream = File.OpenRead(file);
86+
using var peReader = new PEReader(pdbStream);
87+
try
88+
{
89+
// Check if pdb is embedded
90+
if (peReader.ReadDebugDirectory().Any(entry => entry.Type == DebugDirectoryEntryType.EmbeddedPortablePdb))
91+
{
92+
continue;
93+
}
94+
95+
var debugDirectory = peReader.ReadDebugDirectory().First(entry => entry.Type == DebugDirectoryEntryType.CodeView);
96+
var codeViewData = peReader.ReadCodeViewDebugDirectoryData(debugDirectory);
97+
guid = $"{codeViewData.Guid.ToString("N").Replace("-", string.Empty)}";
98+
}
99+
catch (Exception e) when (e is BadImageFormatException || e is InvalidOperationException)
100+
{
101+
// Ignore binaries without debug info
102+
continue;
103+
}
104+
105+
if (guid != string.Empty)
106+
{
107+
if (!allPdbGuids.ContainsKey(guid))
108+
{
109+
filesWithoutPDBs.Add(file.Substring(SdkLayoutPath.Length + 1));
110+
}
111+
else
112+
{
113+
// Copy matching pdb to symbols path, preserving sdk binary's hierarchy
114+
string sourcePath = (string)allPdbGuids[guid]!;
115+
string destinationPath =
116+
file.Replace(SdkLayoutPath, SdkSymbolsLayoutPath)
117+
.Replace(Path.GetFileName(file), Path.GetFileName(sourcePath));
118+
119+
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
120+
File.Copy(sourcePath, destinationPath, true);
121+
}
122+
}
123+
}
124+
}
125+
126+
return filesWithoutPDBs;
127+
}
128+
129+
public Hashtable IndexAllSymbols()
130+
{
131+
Hashtable allPdbGuids = new Hashtable();
132+
133+
foreach (string file in Directory.GetFiles(AllSymbolsPath, "*.pdb", SearchOption.AllDirectories))
134+
{
135+
using var pdbFileStream = File.OpenRead(file);
136+
var metadataProvider = MetadataReaderProvider.FromPortablePdbStream(pdbFileStream);
137+
var metadataReader = metadataProvider.GetMetadataReader();
138+
if (metadataReader.DebugMetadataHeader == null)
139+
{
140+
continue;
141+
}
142+
143+
var id = new BlobContentId(metadataReader.DebugMetadataHeader.Id);
144+
string guid = $"{id.Guid:N}";
145+
if (!string.IsNullOrEmpty(guid) && !allPdbGuids.ContainsKey(guid))
146+
{
147+
allPdbGuids.Add(guid, file);
148+
}
149+
}
150+
151+
return allPdbGuids;
152+
}
153+
}
154+
}

0 commit comments

Comments
 (0)
0