Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit d30ed54

Browse files
criemencriemen
authored
Merge branch 'main' into criemen/bazel-csharp
2 parents 303dc20 + 5fa1b57 commit d30ed54

145 files changed

Lines changed: 3350 additions & 5181 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.bazelrc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
1111
build --repo_env=CC=clang --repo_env=CXX=clang++
1212

1313
build:linux --cxxopt=-std=c++20
14-
build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
14+
# we currently cannot built the swift extractor for ARM
15+
build:macos --cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64
1516
build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
1617

1718
# this requires developer mode, but is required to have pack installer functioning

‎cpp/ql/lib/change-notes/2024-05-23-Version1.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
category: breaking
3+
---
4+
* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

‎cpp/ql/lib/qlpack.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: codeql/cpp-all
2-
version: 0.13.2-dev
2+
version: 1.0.0-dev
33
groups: cpp
44
dbscheme: semmlecode.cpp.dbscheme
55
extractor: cpp

‎cpp/ql/src/change-notes/2024-05-23-Version1.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
category: breaking
3+
---
4+
* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

‎cpp/ql/src/qlpack.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: codeql/cpp-queries
2-
version: 0.9.13-dev
2+
version: 1.0.0-dev
33
groups:
44
- cpp
55
- queries

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@ internal class EnvironmentVariableNames
6060
/// </summary>
6161
public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
6262

63+
/// <summary>
64+
/// Controls whether to include NuGet feeds from nuget.config files in the fallback restore logic.
65+
/// </summary>
66+
public const string AddNugetConfigFeedsToFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK_INCLUDE_NUGET_CONFIG_FEEDS";
67+
6368
/// <summary>
6469
/// Specifies the path to the nuget executable to be used for package restoration.
6570
/// </summary>

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,14 @@ public HashSet<AssemblyLookupLocation> Restore()
9898
logger.LogInfo($"Checking NuGet feed responsiveness: {checkNugetFeedResponsiveness}");
9999
compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", checkNugetFeedResponsiveness ? "1" : "0"));
100100

101+
HashSet<string>? explicitFeeds = null;
102+
101103
try
102104
{
103-
if (checkNugetFeedResponsiveness && !CheckFeeds())
105+
if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds))
104106
{
105107
// todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
106-
var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds();
108+
var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds(explicitFeeds);
107109
return unresponsiveMissingPackageLocation is null
108110
? []
109111
: [unresponsiveMissingPackageLocation];
@@ -163,7 +165,7 @@ public HashSet<AssemblyLookupLocation> Restore()
163165
LogAllUnusedPackages(dependencies);
164166

165167
var missingPackageLocation = checkNugetFeedResponsiveness
166-
? DownloadMissingPackagesFromSpecificFeeds()
168+
? DownloadMissingPackagesFromSpecificFeeds(explicitFeeds)
167169
: DownloadMissingPackages();
168170

169171
if (missingPackageLocation is not null)
@@ -173,13 +175,24 @@ public HashSet<AssemblyLookupLocation> Restore()
173175
return assemblyLookupLocations;
174176
}
175177

176-
private List<string> GetReachableFallbackNugetFeeds()
178+
private List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
177179
{
178180
var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
179181
if (fallbackFeeds.Count == 0)
180182
{
181183
fallbackFeeds.Add(PublicNugetOrgFeed);
182-
logger.LogInfo($"No fallback Nuget feeds specified. Using default feed: {PublicNugetOrgFeed}");
184+
logger.LogInfo($"No fallback Nuget feeds specified. Adding default feed: {PublicNugetOrgFeed}");
185+
186+
var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
187+
logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
188+
189+
if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0)
190+
{
191+
// There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those.
192+
// But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer.
193+
fallbackFeeds.UnionWith(feedsFromNugetConfigs);
194+
logger.LogInfo($"Using Nuget feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}");
195+
}
183196
}
184197

185198
logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}");
@@ -194,6 +207,8 @@ private List<string> GetReachableFallbackNugetFeeds()
194207
logger.LogInfo($"Reachable fallback Nuget feeds: {string.Join(", ", reachableFallbackFeeds.OrderBy(f => f))}");
195208
}
196209

210+
compilationInfoContainer.CompilationInfos.Add(("Reachable fallback Nuget feed count", reachableFallbackFeeds.Count.ToString()));
211+
197212
return reachableFallbackFeeds;
198213
}
199214

@@ -272,9 +287,9 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
272287
compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
273288
}
274289

275-
private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds()
290+
private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(HashSet<string>? feedsFromNugetConfigs)
276291
{
277-
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
292+
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs);
278293
if (reachableFallbackFeeds.Count > 0)
279294
{
280295
return DownloadMissingPackages(fallbackNugetFeeds: reachableFallbackFeeds);
@@ -623,10 +638,10 @@ private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount,
623638
return (timeoutMilliSeconds, tryCount);
624639
}
625640

626-
private bool CheckFeeds()
641+
private bool CheckFeeds(out HashSet<string> explicitFeeds)
627642
{
628643
logger.LogInfo("Checking Nuget feeds...");
629-
var (explicitFeeds, allFeeds) = GetAllFeeds();
644+
(explicitFeeds, var allFeeds) = GetAllFeeds();
630645

631646
var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
632647
.ToHashSet() ?? [];

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,41 +11,35 @@ internal class Compilation : CachedEntity<object>
1111
{
1212
internal readonly ConcurrentDictionary<string, int> messageCounts = new();
1313

14-
private static (string Cwd, string[] Args) settings;
15-
private static int hashCode;
16-
17-
public static (string Cwd, string[] Args) Settings
18-
{
19-
get { return settings; }
20-
set
21-
{
22-
settings = value;
23-
hashCode = settings.Cwd.GetHashCode();
24-
for (var i = 0; i < settings.Args.Length; i++)
25-
{
26-
hashCode = HashCode.Combine(hashCode, settings.Args[i].GetHashCode());
27-
}
28-
}
29-
}
14+
private readonly string cwd;
15+
private readonly string[] args;
16+
private readonly int hashCode;
3017

3118
#nullable disable warnings
3219
private Compilation(Context cx) : base(cx, null)
3320
{
21+
cwd = cx.Extractor.Cwd;
22+
args = cx.Extractor.Args;
23+
hashCode = cwd.GetHashCode();
24+
for (var i = 0; i < args.Length; i++)
25+
{
26+
hashCode = HashCode.Combine(hashCode, args[i].GetHashCode());
27+
}
3428
}
3529
#nullable restore warnings
3630

3731
public override void Populate(TextWriter trapFile)
3832
{
3933
var assembly = Assembly.CreateOutputAssembly(Context);
4034

41-
trapFile.compilations(this, FileUtils.ConvertToUnix(Compilation.Settings.Cwd));
35+
trapFile.compilations(this, FileUtils.ConvertToUnix(cwd));
4236
trapFile.compilation_assembly(this, assembly);
4337

4438
// Arguments
4539
var expandedIndex = 0;
46-
for (var i = 0; i < Compilation.Settings.Args.Length; i++)
40+
for (var i = 0; i < args.Length; i++)
4741
{
48-
var arg = Compilation.Settings.Args[i];
42+
var arg = args[i];
4943
trapFile.compilation_args(this, i, arg);
5044

5145
if (CommandLineExtensions.IsFileArgument(arg))

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,8 @@ public static ExitCode Run(string[] args)
9797
stopwatch.Start();
9898

9999
var options = Options.CreateWithEnvironment(args);
100-
Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), options.CompilerArguments.ToArray());
100+
var workingDirectory = Directory.GetCurrentDirectory();
101+
var compilerArgs = options.CompilerArguments.ToArray();
101102

102103
using var logger = MakeLogger(options.Verbosity, options.Console);
103104

@@ -123,15 +124,15 @@ public static ExitCode Run(string[] args)
123124

124125
var compilerArguments = CSharpCommandLineParser.Default.Parse(
125126
compilerVersion.ArgsWithResponse,
126-
Entities.Compilation.Settings.Cwd,
127+
workingDirectory,
127128
compilerVersion.FrameworkPath,
128129
compilerVersion.AdditionalReferenceDirectories
129130
);
130131

131132
if (compilerArguments is null)
132133
{
133134
var sb = new StringBuilder();
134-
sb.Append(" Failed to parse command line: ").AppendList(" ", Entities.Compilation.Settings.Args);
135+
sb.Append(" Failed to parse command line: ").AppendList(" ", compilerArgs);
135136
logger.Log(Severity.Error, sb.ToString());
136137
++analyser.CompilationErrors;
137138
return ExitCode.Failed;
@@ -143,7 +144,7 @@ public static ExitCode Run(string[] args)
143144
return ExitCode.Ok;
144145
}
145146

146-
return AnalyseTracing(analyser, compilerArguments, options, canonicalPathCache, stopwatch);
147+
return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, canonicalPathCache, stopwatch);
147148
}
148149
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
149150
{
@@ -376,6 +377,8 @@ public static ExitCode Analyse(Stopwatch stopwatch, Analyser analyser, CommonOpt
376377
}
377378

378379
private static ExitCode AnalyseTracing(
380+
string cwd,
381+
string[] args,
379382
TracingAnalyser analyser,
380383
CSharpCommandLineArguments compilerArguments,
381384
Options options,
@@ -420,7 +423,7 @@ private static ExitCode AnalyseTracing(
420423
.WithMetadataImportOptions(MetadataImportOptions.All)
421424
);
422425
},
423-
(compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation),
426+
(compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation, cwd, args),
424427
() => { });
425428
}
426429

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,10 @@ public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyT
1616
public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options)
1717
{
1818
compilation = compilationIn;
19-
extractor = new StandaloneExtractor(outputPath, compilationInfos, Logger, PathTransformer, options);
19+
extractor = new StandaloneExtractor(Directory.GetCurrentDirectory(), outputPath, compilationInfos, Logger, PathTransformer, options);
2020
this.options = options;
2121
LogExtractorInfo(Extraction.Extractor.Version);
2222
SetReferencePaths();
23-
24-
Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), Array.Empty<string>());
2523
}
2624

2725
#nullable disable warnings

0 commit comments

Comments
 (0)