Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e1158bb

Browse files
calumgrantcalumgrant
committed
C#: Store compilations, compiler diagnostics and performance in the database.
1 parent 4cc23cc commit e1158bb

17 files changed

Lines changed: 484 additions & 15 deletions

File tree

‎csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger
139139
trapFile = trapWriter.TrapFile;
140140
if (nocache || !System.IO.File.Exists(trapFile))
141141
{
142-
var cx = new Extraction.Context(extractor, null, trapWriter, null);
142+
var cx = extractor.CreateContext(null, trapWriter, null);
143143
ExtractCIL(cx, assemblyPath, extractPdbs);
144144
extracted = true;
145145
}

‎csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,38 @@ bool FileIsCached(string src, string dest)
215215
return options.Cache && FileIsUpToDate(src, dest);
216216
}
217217

218+
/// <summary>
219+
/// Extracts compilation-wide entities, such as compilations and compiler diagnostics.
220+
/// </summary>
221+
public void AnalyseCompilation(string cwd, string[] args)
222+
{
223+
extractionTasks.Add(() => DoAnalyseCompilation(cwd, args));
224+
}
225+
226+
Entities.Compilation compilationEntity;
227+
IDisposable compilationTrapFile;
228+
229+
void DoAnalyseCompilation(string cwd, string[] args)
230+
{
231+
try
232+
{
233+
var assemblyPath = extractor.OutputPath;
234+
var assembly = compilation.Assembly;
235+
var projectLayout = layout.LookupProjectOrDefault(assemblyPath);
236+
var trapWriter = projectLayout.CreateTrapWriter(Logger, assemblyPath, true);
237+
compilationTrapFile = trapWriter; // Dispose later
238+
var cx = extractor.CreateContext(compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath, true));
239+
240+
compilationEntity = new Entities.Compilation(cx, cwd, args);
241+
}
242+
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
243+
{
244+
Logger.Log(Severity.Error, " Unhandled exception analyzing {0}: {1}", "compilation", ex);
245+
}
246+
}
247+
248+
public void LogPerformance(Entities.Performance p) => compilationEntity.PopulatePerformance(p);
249+
218250
/// <summary>
219251
/// Extract an assembly to a new trap file.
220252
/// If the trap file exists, skip extraction to avoid duplicating
@@ -258,7 +290,7 @@ void DoAnalyseAssembly(PortableExecutableReference r)
258290

259291
if (assembly != null)
260292
{
261-
var cx = new Context(extractor, c, trapWriter, new AssemblyScope(assembly, assemblyPath));
293+
var cx = extractor.CreateContext(c, trapWriter, new AssemblyScope(assembly, assemblyPath, false));
262294

263295
foreach (var module in assembly.Modules)
264296
{
@@ -344,7 +376,7 @@ void DoExtractTree(SyntaxTree tree)
344376

345377
if (!upToDate)
346378
{
347-
Context cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree));
379+
Context cx = extractor.CreateContext(compilation.Clone(), trapWriter, new SourceScope(tree));
348380
Populators.CompilationUnit.Extract(cx, tree.GetRoot());
349381
cx.PopulateAll();
350382
cx.ExtractComments(cx.CommentGenerator);
@@ -373,6 +405,8 @@ public void PerformExtraction(int numberOfThreads)
373405

374406
public void Dispose()
375407
{
408+
compilationTrapFile?.Dispose();
409+
376410
stopWatch.Stop();
377411
Logger.Log(Severity.Info, " Peak working set = {0} MB", Process.GetCurrentProcess().PeakWorkingSet64 / (1024 * 1024));
378412

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilation.cs‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Diagnostics;
4+
using System.Linq;
5+
using System.Text;
6+
7+
namespace Semmle.Extraction.CSharp.Entities
8+
{
9+
class Compilation : FreshEntity
10+
{
11+
public Compilation(Context cx, string cwd, string[] args) : base(cx)
12+
{
13+
Extraction.Entities.Assembly.CreateOutputAssembly(cx);
14+
15+
cx.Emit(Tuples.compilations(this, Extraction.Entities.File.PathAsDatabaseString(cwd)));
16+
17+
// Arguments
18+
int index = 0;
19+
foreach(var arg in args)
20+
{
21+
cx.Emit(Tuples.compilation_args(this, index, args[index]));
22+
index++;
23+
}
24+
25+
// Files
26+
index = 0;
27+
foreach(var file in cx.Compilation.SyntaxTrees.Select(tree => Extraction.Entities.File.Create(cx, tree.FilePath)))
28+
{
29+
cx.Emit(Tuples.compilation_compiling_files(this, index++, file));
30+
}
31+
32+
// Diagnostics
33+
index = 0;
34+
foreach(var d in cx.Compilation.GetDiagnostics())
35+
{
36+
var d2 = new Diagnostic(cx, d);
37+
cx.Emit(Tuples.diagnostic_for(d2, this, 0, index++));
38+
}
39+
}
40+
41+
public void PopulatePerformance(Performance p)
42+
{
43+
int index = 0;
44+
foreach(float metric in p.Metrics)
45+
{
46+
cx.Emit(Tuples.compilation_time(this, -1, index++, metric));
47+
}
48+
}
49+
50+
public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel;
51+
}
52+
class Diagnostic : FreshEntity
53+
{
54+
public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel;
55+
56+
public Diagnostic(Context cx, Microsoft.CodeAnalysis.Diagnostic diag) : base(cx)
57+
{
58+
cx.Emit(Tuples.diagnostics(this, (int)diag.Severity, diag.Id, diag.Descriptor.Title.ToString(),
59+
diag.GetMessage(), Extraction.Entities.Location.Create(cx, diag.Location)));
60+
}
61+
}
62+
63+
public struct Timings
64+
{
65+
public TimeSpan Elapsed, Cpu, User;
66+
}
67+
68+
/// <summary>
69+
/// The various performance measures that we log.
70+
/// </summary>
71+
public struct Performance
72+
{
73+
public Timings Compiler, Extractor;
74+
public long PeakWorkingSet;
75+
76+
/// <summary>
77+
/// These are in database order (0 indexed)
78+
/// </summary>
79+
public IEnumerable<float> Metrics
80+
{
81+
get
82+
{
83+
yield return (float)Compiler.Cpu.TotalSeconds;
84+
yield return (float)Compiler.Elapsed.TotalSeconds;
85+
yield return (float)Extractor.Cpu.TotalSeconds;
86+
yield return (float)Extractor.Elapsed.TotalSeconds;
87+
88+
yield return (float)Compiler.User.TotalSeconds;
89+
yield return (float)Extractor.User.TotalSeconds;
90+
yield return PeakWorkingSet / 1024.0f / 1024.0f;
91+
}
92+
}
93+
}
94+
}

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -118,8 +118,8 @@ public static ExitCode Run(string[] args)
118118
compilerArguments.Encoding,
119119
syntaxTrees);
120120

121-
var sw = new Stopwatch();
122-
sw.Start();
121+
var sw1 = new Stopwatch();
122+
sw1.Start();
123123

124124
Parallel.Invoke(
125125
new ParallelOptions { MaxDegreeOfParallelism = commandLineArguments.Threads },
@@ -148,20 +148,36 @@ public static ExitCode Run(string[] args)
148148
);
149149

150150
analyser.Initialize(compilerArguments, compilation, commandLineArguments, compilerVersion.ArgsWithResponse);
151+
analyser.AnalyseCompilation(cwd, args);
151152
analyser.AnalyseReferences();
152153

153154
foreach (var tree in compilation.SyntaxTrees)
154155
{
155156
analyser.AnalyseTree(tree);
156157
}
157158

158-
sw.Stop();
159-
logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed);
159+
var currentProcess = Process.GetCurrentProcess();
160+
var cpuTime1 = currentProcess.TotalProcessorTime;
161+
var userTime1 = currentProcess.UserProcessorTime;
162+
sw1.Stop();
163+
logger.Log(Severity.Info, " Models constructed in {0}", sw1.Elapsed);
160164

161-
sw.Restart();
165+
var sw2 = new Stopwatch();
166+
sw2.Start();
162167
analyser.PerformExtraction(commandLineArguments.Threads);
163-
sw.Stop();
164-
logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed);
168+
sw2.Stop();
169+
var cpuTime2 = currentProcess.TotalProcessorTime;
170+
var userTime2 = currentProcess.UserProcessorTime;
171+
172+
var performance = new Entities.Performance()
173+
{
174+
Compiler = new Entities.Timings() { Elapsed = sw1.Elapsed, Cpu = cpuTime1, User = userTime1 },
175+
Extractor = new Entities.Timings() { Elapsed = sw2.Elapsed, Cpu = cpuTime2 - cpuTime1, User = userTime2 - userTime1 },
176+
PeakWorkingSet = currentProcess.PeakWorkingSet64
177+
};
178+
179+
analyser.LogPerformance(performance);
180+
logger.Log(Severity.Info, " Extraction took {0}", sw2.Elapsed);
165181

166182
return analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors;
167183
}

‎csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@ internal static class Tuples
4141

4242
internal static Tuple commentline_location(CommentLine commentLine, Location location) => new Tuple("commentline_location", commentLine, location);
4343

44+
internal static Tuple compilation_args(Compilation compilation, int index, string arg) => new Tuple("compilation_args", compilation, index, arg);
45+
46+
internal static Tuple compilation_compiling_files(Compilation compilation, int index, File file) => new Tuple("compilation_compiling_files", compilation, index, file);
47+
48+
internal static Tuple compilation_time(Compilation compilation, int num, int index, float metric) => new Tuple("compilation_time", compilation, num, index, metric);
49+
50+
internal static Tuple compilations(Compilation compilation, string cwd) => new Tuple("compilations", compilation, cwd);
51+
4452
internal static Tuple compiler_generated(IEntity entity) => new Tuple("compiler_generated", entity);
4553

4654
internal static Tuple conditional_access(Expression access) => new Tuple("conditional_access", access);
@@ -59,6 +67,11 @@ internal static class Tuples
5967

6068
internal static Tuple destructors(Destructor destructor, string name, Type containingType, Destructor original) => new Tuple("destructors", destructor, name, containingType, original);
6169

70+
internal static Tuple diagnostic_for(Diagnostic diag, Compilation comp, int fileNo, int index) => new Tuple("diagnostic_for", diag, comp, fileNo, index);
71+
72+
internal static Tuple diagnostics(Diagnostic diag, int severity, string errorTag, string errorMessage, string fullErrorMessage, Location location) =>
73+
new Tuple("diagnostics", diag, severity, errorTag, errorMessage, fullErrorMessage, location);
74+
6275
internal static Tuple dynamic_member_name(Expression e, string name) => new Tuple("dynamic_member_name", e, name);
6376

6477
internal static Tuple enum_underlying_type(Type @enum, Type type) => new Tuple("enum_underlying_type", @enum, type);

‎csharp/extractor/Semmle.Extraction/Context.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,8 @@ public Context(IExtractor e, Compilation c, TrapWriter trapWriter, IExtractionSc
247247
TrapWriter = trapWriter;
248248
}
249249

250+
public bool IsGlobalContext => Scope.IsGlobalScope;
251+
250252
public readonly ICommentGenerator CommentGenerator = new CommentProcessor();
251253

252254
readonly IExtractionScope Scope;

‎csharp/extractor/Semmle.Extraction/Entities/Assembly.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ public override void Populate()
3434
}
3535
}
3636

37-
public override bool NeedsPopulation => true;
37+
public override bool NeedsPopulation =>
38+
assembly != Context.Compilation.Assembly || !Context.IsGlobalContext;
3839

3940
public override int GetHashCode() =>
4041
symbol == null ? 91187354 : symbol.GetHashCode();

‎csharp/extractor/Semmle.Extraction/Entities/Folder.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public override void Populate()
2727
// On Windows: System.IO.DirectoryInfo.Name returns "L:\"
2828
string shortName = symbol.Parent == null ? "" : symbol.Name;
2929

30-
Context.Emit(Tuples.folders(this, Semmle.Extraction.Entities.File.PathAsDatabaseString(Path), shortName));
30+
Context.Emit(Tuples.folders(this, File.PathAsDatabaseString(Path), shortName));
3131
if (symbol.Parent != null)
3232
{
3333
Context.Emit(Tuples.containerparent(Create(Context, symbol.Parent), this));

‎csharp/extractor/Semmle.Extraction/Entities/Location.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ public abstract class Location : CachedEntity<Microsoft.CodeAnalysis.Location>
66
public Location(Context cx, Microsoft.CodeAnalysis.Location init)
77
: base(cx, init) { }
88

9-
internal static Location Create(Context cx, Microsoft.CodeAnalysis.Location loc) =>
9+
public static Location Create(Context cx, Microsoft.CodeAnalysis.Location loc) =>
1010
loc == null ? GeneratedLocation.Create(cx)
1111
: loc.IsInSource ? SourceLocation.Create(cx, loc)
1212
: Assembly.Create(cx, loc);

‎csharp/extractor/Semmle.Extraction/ExtractionScope.cs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ public interface IExtractionScope
2323
/// </summary>
2424
/// <param name="path">The path to populate.</param>
2525
bool InFileScope(string path);
26+
27+
bool IsGlobalScope { get; }
2628
}
2729

2830
/// <summary>
@@ -33,15 +35,18 @@ public class AssemblyScope : IExtractionScope
3335
readonly IAssemblySymbol assembly;
3436
readonly string filepath;
3537

36-
public AssemblyScope(IAssemblySymbol symbol, string path)
38+
public AssemblyScope(IAssemblySymbol symbol, string path, bool isOutput)
3739
{
3840
assembly = symbol;
3941
filepath = path;
42+
IsGlobalScope = isOutput;
4043
}
4144

45+
public bool IsGlobalScope { get; }
46+
4247
public bool InFileScope(string path) => path == filepath;
4348

44-
public bool InScope(ISymbol symbol) => Equals(symbol.ContainingAssembly, assembly);
49+
public bool InScope(ISymbol symbol) => Equals(symbol.ContainingAssembly, assembly) || Equals(symbol, assembly);
4550
}
4651

4752
/// <summary>
@@ -56,6 +61,8 @@ public SourceScope(SyntaxTree tree)
5661
sourceTree = tree;
5762
}
5863

64+
public bool IsGlobalScope => false;
65+
5966
public bool InFileScope(string path) => path == sourceTree.FilePath;
6067

6168
public bool InScope(ISymbol symbol) => symbol.Locations.Any(loc => loc.SourceTree == sourceTree);

0 commit comments

Comments
 (0)