Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 47f9d4b

Browse files
davidwrightondavidwrighton
authored
Generate deterministic timestamp (dotnet#40122)
- Port the logic from the Roslyn compiler that handles deterministic timestamp generation - Enable for composite images This allows matching up generated pdb files with the composite image to work correctly in all known cases.
1 parent 65b8cf2 commit 47f9d4b

File tree

6 files changed

+285
-11
lines changed

6 files changed

+285
-11
lines changed

‎src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/CodeGen/ReadyToRunObjectWriter.cs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using System.Diagnostics;
77
using System.IO;
88
using System.Linq;
9+
using System.Reflection.Metadata;
910
using System.Reflection.PortableExecutable;
1011

1112
using ILCompiler.DependencyAnalysis.ReadyToRun;
@@ -102,8 +103,9 @@ public void EmitPortableExecutable()
102103
stopwatch.Start();
103104

104105
PEHeaderBuilder headerBuilder;
105-
int timeDateStamp;
106+
int? timeDateStamp;
106107
ISymbolNode r2rHeaderExportSymbol;
108+
Func<IEnumerable<Blob>, BlobContentId> peIdProvider = null;
107109

108110
if (_nodeFactory.CompilationModuleGroup.IsCompositeBuildMode && _componentModule == null)
109111
{
@@ -112,8 +114,8 @@ public void EmitPortableExecutable()
112114
dllCharacteristics: default(DllCharacteristics),
113115
Subsystem.Unknown,
114116
_nodeFactory.Target);
115-
// TODO: generate a non-zero timestamp: https://github.com/dotnet/runtime/issues/32507
116-
timeDateStamp = 0;
117+
peIdProvider = new Func<IEnumerable<Blob>, BlobContentId>(content => BlobContentId.FromHash(CryptographicHashProvider.ComputeSourceHash(content)));
118+
timeDateStamp = null;
117119
r2rHeaderExportSymbol = _nodeFactory.Header;
118120
}
119121
else
@@ -135,7 +137,8 @@ public void EmitPortableExecutable()
135137
r2rHeaderExportSymbol,
136138
Path.GetFileName(_objectFilePath),
137139
getRuntimeFunctionsTable,
138-
_customPESectionAlignment);
140+
_customPESectionAlignment,
141+
peIdProvider);
139142

140143
NativeDebugDirectoryEntryNode nativeDebugDirectoryEntryNode = null;
141144
ISymbolDefinitionNode firstImportThunk = null;

‎src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/Compiler/CryptographicHashProvider.cs

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
#nullable enable
5+
6+
using System;
7+
using System.Collections.Generic;
8+
using System.Collections.Immutable;
9+
using System.IO;
10+
using System.Linq;
11+
using System.Reflection;
12+
using System.Reflection.Metadata;
13+
using System.Security.Cryptography;
14+
15+
namespace ILCompiler
16+
{
17+
/// <summary>
18+
/// Specifies a hash algorithms used for hashing source files.
19+
/// </summary>
20+
public enum SourceHashAlgorithm
21+
{
22+
/// <summary>
23+
/// No algorithm specified.
24+
/// </summary>
25+
None = 0,
26+
27+
/// <summary>
28+
/// Secure Hash Algorithm 1.
29+
/// </summary>
30+
Sha1 = 1,
31+
32+
/// <summary>
33+
/// Secure Hash Algorithm 2 with a hash size of 256 bits.
34+
/// </summary>
35+
Sha256 = 2,
36+
}
37+
38+
internal static class SourceHashAlgorithmUtils
39+
{
40+
public const SourceHashAlgorithm DefaultContentHashAlgorithm = SourceHashAlgorithm.Sha256;
41+
}
42+
43+
internal abstract class CryptographicHashProvider
44+
{
45+
private ImmutableArray<byte> _lazySHA1Hash;
46+
private ImmutableArray<byte> _lazySHA256Hash;
47+
private ImmutableArray<byte> _lazySHA384Hash;
48+
private ImmutableArray<byte> _lazySHA512Hash;
49+
private ImmutableArray<byte> _lazyMD5Hash;
50+
51+
internal abstract ImmutableArray<byte> ComputeHash(HashAlgorithm algorithm);
52+
53+
internal ImmutableArray<byte> GetHash(AssemblyHashAlgorithm algorithmId)
54+
{
55+
using (HashAlgorithm? algorithm = TryGetAlgorithm(algorithmId))
56+
{
57+
// ERR_CryptoHashFailed has already been reported:
58+
if (algorithm == null)
59+
{
60+
return ImmutableArray.Create<byte>();
61+
}
62+
63+
switch (algorithmId)
64+
{
65+
case AssemblyHashAlgorithm.None:
66+
case AssemblyHashAlgorithm.Sha1:
67+
return GetHash(ref _lazySHA1Hash, algorithm);
68+
69+
case AssemblyHashAlgorithm.Sha256:
70+
return GetHash(ref _lazySHA256Hash, algorithm);
71+
72+
case AssemblyHashAlgorithm.Sha384:
73+
return GetHash(ref _lazySHA384Hash, algorithm);
74+
75+
case AssemblyHashAlgorithm.Sha512:
76+
return GetHash(ref _lazySHA512Hash, algorithm);
77+
78+
case AssemblyHashAlgorithm.MD5:
79+
return GetHash(ref _lazyMD5Hash, algorithm);
80+
81+
default:
82+
throw new ArgumentException("algorithmId");
83+
}
84+
}
85+
}
86+
87+
internal static int GetHashSize(SourceHashAlgorithm algorithmId)
88+
{
89+
switch (algorithmId)
90+
{
91+
case SourceHashAlgorithm.Sha1:
92+
return 160 / 8;
93+
94+
case SourceHashAlgorithm.Sha256:
95+
return 256 / 8;
96+
97+
default:
98+
throw new ArgumentException("algorithmId");
99+
}
100+
}
101+
102+
internal static HashAlgorithm? TryGetAlgorithm(SourceHashAlgorithm algorithmId)
103+
{
104+
switch (algorithmId)
105+
{
106+
case SourceHashAlgorithm.Sha1:
107+
return SHA1.Create();
108+
109+
case SourceHashAlgorithm.Sha256:
110+
return SHA256.Create();
111+
112+
default:
113+
return null;
114+
}
115+
}
116+
117+
internal static HashAlgorithmName GetAlgorithmName(SourceHashAlgorithm algorithmId)
118+
{
119+
switch (algorithmId)
120+
{
121+
case SourceHashAlgorithm.Sha1:
122+
return HashAlgorithmName.SHA1;
123+
124+
case SourceHashAlgorithm.Sha256:
125+
return HashAlgorithmName.SHA256;
126+
127+
default:
128+
throw new ArgumentException("algorithmId");
129+
}
130+
}
131+
132+
internal static HashAlgorithm? TryGetAlgorithm(AssemblyHashAlgorithm algorithmId)
133+
{
134+
switch (algorithmId)
135+
{
136+
case AssemblyHashAlgorithm.None:
137+
case AssemblyHashAlgorithm.Sha1:
138+
return SHA1.Create();
139+
140+
case AssemblyHashAlgorithm.Sha256:
141+
return SHA256.Create();
142+
143+
case AssemblyHashAlgorithm.Sha384:
144+
return SHA384.Create();
145+
146+
case AssemblyHashAlgorithm.Sha512:
147+
return SHA512.Create();
148+
149+
case AssemblyHashAlgorithm.MD5:
150+
return MD5.Create();
151+
152+
default:
153+
return null;
154+
}
155+
}
156+
157+
internal static bool IsSupportedAlgorithm(AssemblyHashAlgorithm algorithmId)
158+
{
159+
switch (algorithmId)
160+
{
161+
case AssemblyHashAlgorithm.None:
162+
case AssemblyHashAlgorithm.Sha1:
163+
case AssemblyHashAlgorithm.Sha256:
164+
case AssemblyHashAlgorithm.Sha384:
165+
case AssemblyHashAlgorithm.Sha512:
166+
case AssemblyHashAlgorithm.MD5:
167+
return true;
168+
169+
default:
170+
return false;
171+
}
172+
}
173+
174+
private ImmutableArray<byte> GetHash(ref ImmutableArray<byte> lazyHash, HashAlgorithm algorithm)
175+
{
176+
if (lazyHash.IsDefault)
177+
{
178+
ImmutableInterlocked.InterlockedCompareExchange(ref lazyHash, ComputeHash(algorithm), default(ImmutableArray<byte>));
179+
}
180+
181+
return lazyHash;
182+
}
183+
184+
internal const int Sha1HashSize = 20;
185+
186+
internal static ImmutableArray<byte> ComputeSha1(Stream stream)
187+
{
188+
if (stream != null)
189+
{
190+
stream.Seek(0, SeekOrigin.Begin);
191+
using (var hashProvider = SHA1.Create())
192+
{
193+
return ImmutableArray.Create(hashProvider.ComputeHash(stream));
194+
}
195+
}
196+
197+
return ImmutableArray<byte>.Empty;
198+
}
199+
200+
internal static ImmutableArray<byte> ComputeSha1(ImmutableArray<byte> bytes)
201+
{
202+
return ComputeSha1(bytes.ToArray());
203+
}
204+
205+
internal static ImmutableArray<byte> ComputeSha1(byte[] bytes)
206+
{
207+
using (var hashProvider = SHA1.Create())
208+
{
209+
return ImmutableArray.Create(hashProvider.ComputeHash(bytes));
210+
}
211+
}
212+
213+
internal static ImmutableArray<byte> ComputeHash(HashAlgorithmName algorithmName, IEnumerable<Blob> bytes)
214+
{
215+
using (var incrementalHash = IncrementalHash.CreateHash(algorithmName))
216+
{
217+
foreach (var blob in bytes)
218+
{
219+
incrementalHash.AppendData(blob.GetBytes());
220+
}
221+
return ImmutableArray.Create(incrementalHash.GetHashAndReset());
222+
}
223+
}
224+
225+
internal static ImmutableArray<byte> ComputeHash(HashAlgorithmName algorithmName, IEnumerable<ArraySegment<byte>> bytes)
226+
{
227+
using (var incrementalHash = IncrementalHash.CreateHash(algorithmName))
228+
{
229+
foreach (var segment in bytes)
230+
{
231+
incrementalHash.AppendData(segment);
232+
}
233+
return ImmutableArray.Create(incrementalHash.GetHashAndReset());
234+
}
235+
}
236+
237+
internal static ImmutableArray<byte> ComputeSourceHash(ImmutableArray<byte> bytes, SourceHashAlgorithm hashAlgorithm = SourceHashAlgorithmUtils.DefaultContentHashAlgorithm)
238+
{
239+
var algorithmName = GetAlgorithmName(hashAlgorithm);
240+
using (var incrementalHash = IncrementalHash.CreateHash(algorithmName))
241+
{
242+
incrementalHash.AppendData(bytes.ToArray());
243+
return ImmutableArray.Create(incrementalHash.GetHashAndReset());
244+
}
245+
}
246+
247+
internal static ImmutableArray<byte> ComputeSourceHash(IEnumerable<Blob> bytes, SourceHashAlgorithm hashAlgorithm = SourceHashAlgorithmUtils.DefaultContentHashAlgorithm)
248+
{
249+
return ComputeHash(GetAlgorithmName(hashAlgorithm), bytes);
250+
}
251+
}
252+
}

‎src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryNode.cs

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,12 @@ public class DebugDirectoryNode : ObjectNode, ISymbolDefinitionNode
2626

2727
private EcmaModule _module;
2828
private NativeDebugDirectoryEntryNode _nativeEntry;
29+
private bool _insertDeterministicEntry;
2930

3031
public DebugDirectoryNode(EcmaModule sourceModule, string outputFileName)
3132
{
3233
_module = sourceModule;
34+
_insertDeterministicEntry = sourceModule == null; // Mark module as deterministic if generating composite image
3335
string pdbNameRoot = Path.GetFileNameWithoutExtension(outputFileName);
3436
if (sourceModule != null)
3537
{
@@ -50,7 +52,7 @@ public DebugDirectoryNode(EcmaModule sourceModule, string outputFileName)
5052

5153
public int Offset => 0;
5254

53-
public int Size => (GetNumDebugDirectoryEntriesInModule() + 1) * ImageDebugDirectorySize;
55+
public int Size => (GetNumDebugDirectoryEntriesInModule() + 1 + (_insertDeterministicEntry ? 1 : 0)) * ImageDebugDirectorySize;
5456

5557
public void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb)
5658
{
@@ -112,8 +114,21 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false)
112114
builder.EmitReloc(entry, RelocType.IMAGE_REL_FILE_ABSOLUTE);
113115
}
114116

117+
// If generating a composite image, emit the deterministic marker
118+
if (_insertDeterministicEntry)
119+
{
120+
builder.EmitUInt(0 /* Characteristics */);
121+
builder.EmitUInt(0);
122+
builder.EmitUShort(0);
123+
builder.EmitUShort(0);
124+
builder.EmitInt((int)DebugDirectoryEntryType.Reproducible);
125+
builder.EmitInt(0);
126+
builder.EmitUInt(0);
127+
builder.EmitUInt(0);
128+
}
129+
115130
// Second, copy existing entries from input module
116-
for(int i = 0; i < numEntries; i++)
131+
for (int i = 0; i < numEntries; i++)
117132
{
118133
builder.EmitUInt(0 /* Characteristics */);
119134
builder.EmitUInt(entries[i].Stamp);

‎src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/IBC/IBCProfileParser.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,8 @@ public ProfileData ParseIBCDataFromModule(EcmaModule ecmaModule)
113113
}
114114
else
115115
{
116-
_logger.Writer.WriteLine($"Token {0:x} does not refer to a method");
116+
if (_logger.IsVerbose)
117+
_logger.Writer.WriteLine($"Token {(int)entry.Token:x} does not refer to a method");
117118
}
118119
break;
119120

‎src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
<Compile Include="ObjectWriter\MapFileBuilder.cs" />
9999
<Compile Include="CodeGen\ReadyToRunObjectWriter.cs" />
100100
<Compile Include="Compiler\CompilationModuleGroup.ReadyToRun.cs" />
101+
<Compile Include="Compiler\CryptographicHashProvider.cs" />
101102
<Compile Include="Compiler\DependencyAnalysis\AllMethodsOnTypeNode.cs" />
102103
<Compile Include="Compiler\DependencyAnalysis\ArrayOfEmbeddedDataNode.cs" />
103104
<Compile Include="Compiler\DependencyAnalysis\ArrayOfEmbeddedPointersNode.cs" />

‎src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/R2RPEBuilder.cs

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -173,8 +173,9 @@ public R2RPEBuilder(
173173
ISymbolNode r2rHeaderExportSymbol,
174174
string outputFileSimpleName,
175175
Func<RuntimeFunctionsTableNode> getRuntimeFunctionsTable,
176-
int? customPESectionAlignment)
177-
: base(peHeaderBuilder, deterministicIdProvider: null)
176+
int? customPESectionAlignment,
177+
Func<IEnumerable<Blob>, BlobContentId> deterministicIdProvider)
178+
: base(peHeaderBuilder, deterministicIdProvider: deterministicIdProvider)
178179
{
179180
_target = target;
180181
_getRuntimeFunctionsTable = getRuntimeFunctionsTable;
@@ -288,7 +289,7 @@ public int GetSymbolFilePosition(ISymbolNode symbol)
288289
/// </summary>
289290
/// <param name="outputStream">Output stream for the final R2R PE file</param>
290291
/// <param name="timeDateStamp">Timestamp to set in the PE header of the output R2R executable</param>
291-
public void Write(Stream outputStream, int timeDateStamp)
292+
public void Write(Stream outputStream, int? timeDateStamp)
292293
{
293294
BlobBuilder outputPeFile = new BlobBuilder();
294295
Serialize(outputPeFile);
@@ -302,7 +303,8 @@ public void Write(Stream outputStream, int timeDateStamp)
302303

303304
ApplyMachineOSOverride(outputStream);
304305

305-
SetPEHeaderTimeStamp(outputStream, timeDateStamp);
306+
if (timeDateStamp.HasValue)
307+
SetPEHeaderTimeStamp(outputStream, timeDateStamp.Value);
306308

307309
_written = true;
308310
}

0 commit comments

Comments
 (0)