diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs
index cad7d33f472b..be5f137548c4 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs
@@ -1,14 +1,22 @@
using System;
-using System.Diagnostics;
+using System.Collections.Generic;
using System.IO;
using System.Security.Cryptography.X509Certificates;
using Semmle.Util;
using Semmle.Util.Logging;
+using Newtonsoft.Json;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
public class DependabotProxy : IDisposable
{
+ ///
+ /// Represents configurations for package registries.
+ ///
+ /// The type of package registry.
+ /// The URL of the package registry.
+ public record class RegistryConfig(string Type, string URL);
+
private readonly string host;
private readonly string port;
@@ -17,6 +25,10 @@ public class DependabotProxy : IDisposable
///
internal string Address { get; }
///
+ /// The URLs of package registries that are configured for the proxy.
+ ///
+ internal HashSet RegistryURLs { get; }
+ ///
/// The path to the temporary file where the certificate is stored.
///
internal string? CertificatePath { get; private set; }
@@ -67,6 +79,39 @@ public class DependabotProxy : IDisposable
result.Certificate = X509Certificate2.CreateFromPem(cert);
}
+ // Try to obtain the list of private registry URLs.
+ var registryURLs = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyURLs);
+
+ if (!string.IsNullOrWhiteSpace(registryURLs))
+ {
+ try
+ {
+ // The value of the environment variable should be a JSON array of objects, such as:
+ // [ { "type": "nuget_feed", "url": "https://nuget.pkg.github.com/org/index.json" } ]
+ var array = JsonConvert.DeserializeObject>(registryURLs);
+ if (array is not null)
+ {
+ foreach (RegistryConfig config in array)
+ {
+ // The array contains all configured private registries, not just ones for C#.
+ // We ignore the non-C# ones here.
+ if (!config.Type.Equals("nuget_feed"))
+ {
+ logger.LogDebug($"Ignoring registry at '{config.URL}' since it is not of type 'nuget_feed'.");
+ continue;
+ }
+
+ logger.LogInfo($"Found private registry at '{config.URL}'");
+ result.RegistryURLs.Add(config.URL);
+ }
+ }
+ }
+ catch (JsonException ex)
+ {
+ logger.LogError($"Unable to parse '{EnvironmentVariableNames.ProxyURLs}': {ex.Message}");
+ }
+ }
+
return result;
}
@@ -75,6 +120,7 @@ private DependabotProxy(string host, string port)
this.host = host;
this.port = port;
this.Address = $"http://{this.host}:{this.port}";
+ this.RegistryURLs = new HashSet();
}
public void Dispose()
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
index dfabb7446186..49d35c944bd8 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
@@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
-
using Newtonsoft.Json.Linq;
using Semmle.Util;
@@ -77,6 +76,11 @@ private string GetRestoreArgs(RestoreSettings restoreSettings)
args += " /p:EnableWindowsTargeting=true";
}
+ if (restoreSettings.ExtraArgs is not null)
+ {
+ args += $" {restoreSettings.ExtraArgs}";
+ }
+
return args;
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs
index d825e5daeb03..589e72d21265 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs
@@ -89,5 +89,10 @@ internal static class EnvironmentVariableNames
/// Contains the certificate used by the Dependabot proxy.
///
public const string ProxyCertificate = "CODEQL_PROXY_CA_CERTIFICATE";
+
+ ///
+ /// Contains the URLs of private nuget registries as a JSON array.
+ ///
+ public const string ProxyURLs = "CODEQL_PROXY_URLS";
}
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
index 2c10afa80ef2..eec6a2b8d3b2 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
@@ -17,7 +17,7 @@ public interface IDotNet
IList GetNugetFeedsFromFolder(string folderPath);
}
- public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false, bool TargetWindows = false);
+ public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? ExtraArgs = null, string? PathToNugetConfig = null, bool ForceReevaluation = false, bool TargetWindows = false);
public partial record class RestoreResult(bool Success, IList Output)
{
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
index 393e37579b71..d487bc37572a 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
@@ -103,10 +103,11 @@ public HashSet Restore()
compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", checkNugetFeedResponsiveness ? "1" : "0"));
HashSet? explicitFeeds = null;
+ HashSet? allFeeds = null;
try
{
- if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds))
+ if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds, out allFeeds))
{
// todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds(explicitFeeds);
@@ -156,7 +157,7 @@ public HashSet Restore()
var restoredProjects = RestoreSolutions(out var container);
var projects = fileProvider.Projects.Except(restoredProjects);
- RestoreProjects(projects, out var containers);
+ RestoreProjects(projects, allFeeds, out var containers);
var dependencies = containers.Flatten(container);
@@ -260,8 +261,33 @@ private IEnumerable RestoreSolutions(out DependencyContainer dependencie
/// Populates dependencies with the relative paths to the assets files generated by the restore.
///
/// A list of paths to project files.
- private void RestoreProjects(IEnumerable projects, out ConcurrentBag dependencies)
+ private void RestoreProjects(IEnumerable projects, HashSet? configuredSources, out ConcurrentBag dependencies)
{
+ // Conservatively, we only set this to a non-null value if a Dependabot proxy is enabled.
+ // This ensures that we continue to get the old behaviour where feeds are taken from
+ // `nuget.config` files instead of the command-line arguments.
+ string? extraArgs = null;
+
+ if (this.dependabotProxy is not null)
+ {
+ // If the Dependabot proxy is configured, then our main goal is to make `dotnet` aware
+ // of the private registry feeds. However, since providing them as command-line arguments
+ // to `dotnet` ignores other feeds that may be configured, we also need to add the feeds
+ // we have discovered from analysing `nuget.config` files.
+ var sources = configuredSources ?? new();
+ this.dependabotProxy.RegistryURLs.ForEach(url => sources.Add(url));
+
+ // Add package sources. If any are present, they override all sources specified in
+ // the configuration file(s).
+ var feedArgs = new StringBuilder();
+ foreach (string source in sources)
+ {
+ feedArgs.Append($" -s {source}");
+ }
+
+ extraArgs = feedArgs.ToString();
+ }
+
var successCount = 0;
var nugetSourceFailures = 0;
ConcurrentBag collectedDependencies = [];
@@ -276,7 +302,7 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag explicitFeeds)
+ ///
+ /// Checks that we can connect to all Nuget feeds that are explicitly configured in configuration files
+ /// as well as any private package registry feeds that are configured.
+ ///
+ /// Outputs the set of explicit feeds.
+ /// Outputs the set of all feeds (explicit and inherited).
+ /// True if all feeds are reachable or false otherwise.
+ private bool CheckFeeds(out HashSet explicitFeeds, out HashSet allFeeds)
+ {
+ (explicitFeeds, allFeeds) = GetAllFeeds();
+ HashSet feedsToCheck = explicitFeeds;
+
+ // If private package registries are configured for C#, then check those
+ // in addition to the ones that are configured in `nuget.config` files.
+ this.dependabotProxy?.RegistryURLs.ForEach(url => feedsToCheck.Add(url));
+
+ var allFeedsReachable = this.CheckSpecifiedFeeds(feedsToCheck);
+
+ var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet();
+ if (inheritedFeeds.Count > 0)
+ {
+ logger.LogInfo($"Inherited Nuget feeds (not checked for reachability): {string.Join(", ", inheritedFeeds.OrderBy(f => f))}");
+ compilationInfoContainer.CompilationInfos.Add(("Inherited Nuget feed count", inheritedFeeds.Count.ToString()));
+ }
+
+ return allFeedsReachable;
+ }
+
+ ///
+ /// Checks that we can connect to the specified Nuget feeds.
+ ///
+ /// The set of package feeds to check.
+ /// True if all feeds are reachable or false otherwise.
+ private bool CheckSpecifiedFeeds(HashSet feeds)
{
- logger.LogInfo("Checking Nuget feeds...");
- (explicitFeeds, var allFeeds) = GetAllFeeds();
+ logger.LogInfo("Checking that Nuget feeds are reachable...");
var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
.ToHashSet();
@@ -689,7 +747,7 @@ private bool CheckFeeds(out HashSet explicitFeeds)
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
- var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed, initialTimeout, tryCount));
+ var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed, initialTimeout, tryCount));
if (!allFeedsReachable)
{
logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");
@@ -704,14 +762,6 @@ private bool CheckFeeds(out HashSet explicitFeeds)
}
compilationInfoContainer.CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
-
- var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet();
- if (inheritedFeeds.Count > 0)
- {
- logger.LogInfo($"Inherited Nuget feeds (not checked for reachability): {string.Join(", ", inheritedFeeds.OrderBy(f => f))}");
- compilationInfoContainer.CompilationInfos.Add(("Inherited Nuget feed count", inheritedFeeds.Count.ToString()));
- }
-
return allFeedsReachable;
}
@@ -760,23 +810,33 @@ private IEnumerable GetFeeds(Func> getNugetFeeds)
}
// todo: this could be improved.
- // We don't have to get the feeds from each of the folders from below, it would be enought to check the folders that recursively contain the others.
- var allFeeds = nugetConfigs
- .Select(config =>
- {
- try
- {
- return new FileInfo(config).Directory?.FullName;
- }
- catch (Exception exc)
+ HashSet? allFeeds = null;
+
+ if (nugetConfigs.Count > 0)
+ {
+ // We don't have to get the feeds from each of the folders from below, it would be enought to check the folders that recursively contain the others.
+ allFeeds = nugetConfigs
+ .Select(config =>
{
- logger.LogWarning($"Failed to get directory of '{config}': {exc}");
- }
- return null;
- })
- .Where(folder => folder != null)
- .SelectMany(folder => GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folder!)))
- .ToHashSet();
+ try
+ {
+ return new FileInfo(config).Directory?.FullName;
+ }
+ catch (Exception exc)
+ {
+ logger.LogWarning($"Failed to get directory of '{config}': {exc}");
+ }
+ return null;
+ })
+ .Where(folder => folder != null)
+ .SelectMany(folder => GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folder!)))
+ .ToHashSet();
+ }
+ else
+ {
+ // If we haven't found any `nuget.config` files, then obtain a list of feeds from the root source directory.
+ allFeeds = GetFeeds(() => dotnet.GetNugetFeedsFromFolder(this.fileProvider.SourceDir.FullName)).ToHashSet();
+ }
logger.LogInfo($"Found {allFeeds.Count} Nuget feeds (with inherited ones) in nuget.config files: {string.Join(", ", allFeeds.OrderBy(f => f))}");
diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
index c584b607ec8e..904ad04ce82f 100644
--- a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
+++ b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
@@ -123,7 +123,7 @@ public void TestDotnetRestoreProjectToDirectory2()
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
- var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config"));
+ var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, null, "myconfig.config"));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -141,7 +141,7 @@ public void TestDotnetRestoreProjectToDirectory3()
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
- var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config", true));
+ var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, null, "myconfig.config", true));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
diff --git a/csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected b/csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected
index 48cca2534533..ee27a1cd9120 100644
--- a/csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected
+++ b/csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected
@@ -1,6 +1,7 @@
| All Nuget feeds reachable | 1.0 |
| Failed project restore with package source error | 0.0 |
| Failed solution restore with package source error | 0.0 |
+| Inherited Nuget feed count | 1.0 |
| NuGet feed responsiveness checked | 1.0 |
| Project files on filesystem | 1.0 |
| Reachable fallback Nuget feed count | 1.0 |
diff --git a/csharp/ql/integration-tests/all-platforms/standalone_winforms/CompilationInfo.expected b/csharp/ql/integration-tests/all-platforms/standalone_winforms/CompilationInfo.expected
index f87af9b7599d..cf2e7f2db702 100644
--- a/csharp/ql/integration-tests/all-platforms/standalone_winforms/CompilationInfo.expected
+++ b/csharp/ql/integration-tests/all-platforms/standalone_winforms/CompilationInfo.expected
@@ -1,6 +1,7 @@
| All Nuget feeds reachable | 1.0 |
| Failed project restore with package source error | 0.0 |
| Failed solution restore with package source error | 0.0 |
+| Inherited Nuget feed count | 1.0 |
| NuGet feed responsiveness checked | 1.0 |
| Project files on filesystem | 1.0 |
| Reachable fallback Nuget feed count | 1.0 |