From 11faf08ed08b620c6c97c4add6a51bea57cfc833 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Tue, 11 Jun 2024 14:16:35 +0200 Subject: [PATCH 1/6] Remove specific standalone/tracing extractor state classes --- .../Extractor/StandaloneAnalyser.cs | 2 +- .../Extractor/TracingAnalyser.cs | 2 +- .../Semmle.Extraction/Extractor/Extractor.cs | 15 ++++++----- .../Extractor/StandaloneExtractor.cs | 25 ------------------- .../Extractor/TracingExtractor.cs | 25 ------------------- 5 files changed, 11 insertions(+), 58 deletions(-) delete mode 100644 csharp/extractor/Semmle.Extraction/Extractor/StandaloneExtractor.cs delete mode 100644 csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs index 263801e6e8ed..82ea40c2d949 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs @@ -16,7 +16,7 @@ public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyT public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options) { compilation = compilationIn; - extractor = new StandaloneExtractor(Directory.GetCurrentDirectory(), outputPath, compilationInfos, Logger, PathTransformer, options); + extractor = new Extraction.Extractor(Directory.GetCurrentDirectory(), [], outputPath, compilationInfos, Logger, PathTransformer, ExtractorMode.Standalone, options.QlTest); this.options = options; LogExtractorInfo(Extraction.Extractor.Version); SetReferencePaths(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs index c609b2ba100a..43c5562736cd 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs @@ -46,7 +46,7 @@ public void EndInitialize( throw new InternalError("EndInitialize called without BeginInitialize returning true"); this.options = options; this.compilation = compilation; - this.extractor = new TracingExtractor(cwd, args, GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options); + this.extractor = new Extraction.Extractor(cwd, args, GetOutputName(compilation, commandLineArguments), [], Logger, PathTransformer, ExtractorMode.None, options.QlTest); LogDiagnostics(); SetReferencePaths(); diff --git a/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs b/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs index 078d5d53798b..c8da1265559e 100644 --- a/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs @@ -1,6 +1,5 @@ using System.Collections.Generic; using System.Reflection; -using System.IO; using Semmle.Util.Logging; using CompilationInfo = (string key, string value); @@ -9,20 +8,18 @@ namespace Semmle.Extraction /// /// Implementation of the main extractor state. /// - public abstract class Extractor + public class Extractor { public string Cwd { get; init; } public string[] Args { get; init; } - public abstract ExtractorMode Mode { get; } + public ExtractorMode Mode { get; } public string OutputPath { get; } public IEnumerable CompilationInfos { get; } /// /// Creates a new extractor instance for one compilation unit. /// - /// The object used for logging. - /// The object used for path transformations. - protected Extractor(string cwd, string[] args, string outputPath, IEnumerable compilationInfos, ILogger logger, PathTransformer pathTransformer) + public Extractor(string cwd, string[] args, string outputPath, IEnumerable compilationInfos, ILogger logger, PathTransformer pathTransformer, ExtractorMode mode, bool isQlTest) { OutputPath = outputPath; Logger = logger; @@ -30,6 +27,12 @@ protected Extractor(string cwd, string[] args, string outputPath, IEnumerable - /// Creates a new extractor instance for one compilation unit. - /// - /// The object used for logging. - /// The object used for path transformations. - public StandaloneExtractor(string cwd, string outputPath, IEnumerable<(string, string)> compilationInfos, ILogger logger, PathTransformer pathTransformer, CommonOptions options) - : base(cwd, [], outputPath, compilationInfos, logger, pathTransformer) - { - Mode = ExtractorMode.Standalone; - if (options.QlTest) - { - Mode |= ExtractorMode.QlTest; - } - } - } -} diff --git a/csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs b/csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs deleted file mode 100644 index 54230d09ac21..000000000000 --- a/csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Semmle.Util.Logging; - -namespace Semmle.Extraction -{ - public class TracingExtractor : Extractor - { - public override ExtractorMode Mode { get; } - - /// - /// Creates a new extractor instance for one compilation unit. - /// - /// The name of the output DLL/EXE, or null if not specified (standalone extraction). - /// The object used for logging. - /// The object used for path transformations. - public TracingExtractor(string cwd, string[] args, string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) - : base(cwd, args, outputPath, [], logger, pathTransformer) - { - Mode = ExtractorMode.None; - if (options.QlTest) - { - Mode |= ExtractorMode.QlTest; - } - } - } -} From af2a78ea4d3a7bcd3a1d064c51f77a3af779fe26 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 12 Jun 2024 11:12:57 +0200 Subject: [PATCH 2/6] Reduce references to `Extract` class --- .../Extractor.cs | 2 +- .../Entities/NonGeneratedSourceLocation.cs | 22 +-------- .../LineOrSpanDirective.cs | 2 +- .../PragmaChecksumDirective.cs | 2 +- .../Extractor/Analyser.cs | 35 ++++++++++++-- .../Extractor/Context.cs | 21 +++++++++ .../Extractor/Extractor.cs | 15 +++--- .../Extractor/StandaloneAnalyser.cs | 7 +-- .../Extractor/TracingAnalyser.cs | 46 +++++++------------ csharp/extractor/Semmle.Extraction/Context.cs | 2 +- .../Semmle.Extraction/Entities/Base/Entity.cs | 4 +- .../Entities/Base/IEntity.cs | 2 +- .../Semmle.Extraction/Extractor/Extractor.cs | 16 ------- 13 files changed, 87 insertions(+), 89 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs index e578dd4aa31c..a72d04ac8ba4 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs @@ -81,7 +81,7 @@ private static void ExtractStandalone( var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); var pathTransformer = new PathTransformer(canonicalPathCache); - using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer); + using var analyser = new StandaloneAnalyser(pm, logger, pathTransformer, canonicalPathCache, false); try { AnalyseStandalone(analyser, extractionInput, options, pm, stopwatch); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs index a3b7877af4e7..df41abd524d3 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs @@ -27,7 +27,7 @@ public override void Populate(TextWriter trapFile) var mapped = Symbol.GetMappedLineSpan(); if (mapped.HasMappedPath && mapped.IsValid) { - var path = TryAdjustRelativeMappedFilePath(mapped.Path, Position.Path, Context.Extractor.Logger); + var path = Context.TryAdjustRelativeMappedFilePath(mapped.Path, Position.Path); var mappedLoc = Create(Context, Location.Create(path, default, mapped.Span)); trapFile.locations_mapped(this, mappedLoc); @@ -64,25 +64,5 @@ private class SourceLocationFactory : CachedEntityFactory new NonGeneratedSourceLocation(cx, init); } - - public static string TryAdjustRelativeMappedFilePath(string mappedToPath, string mappedFromPath, ILogger logger) - { - if (!Path.IsPathRooted(mappedToPath)) - { - try - { - var fullPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(mappedFromPath)!, mappedToPath)); - logger.LogDebug($"Found relative path in line mapping: '{mappedToPath}', interpreting it as '{fullPath}'"); - - mappedToPath = fullPath; - } - catch (Exception e) - { - logger.LogDebug($"Failed to compute absolute path for relative path in line mapping: '{mappedToPath}': {e}"); - } - } - - return mappedToPath; - } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineOrSpanDirective.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineOrSpanDirective.cs index 6d2362203316..4c4806110f00 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineOrSpanDirective.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineOrSpanDirective.cs @@ -28,7 +28,7 @@ protected override void PopulatePreprocessor(TextWriter trapFile) var path = Symbol.File.ValueText; if (!string.IsNullOrWhiteSpace(path)) { - path = NonGeneratedSourceLocation.TryAdjustRelativeMappedFilePath(path, Symbol.SyntaxTree.FilePath, Context.Extractor.Logger); + path = Context.TryAdjustRelativeMappedFilePath(path, Symbol.SyntaxTree.FilePath); var file = File.Create(Context, path); trapFile.directive_line_file(this, file); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs index 3e0c468d85bc..66eab50331f2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs @@ -12,7 +12,7 @@ private PragmaChecksumDirective(Context cx, PragmaChecksumDirectiveTriviaSyntax protected override void PopulatePreprocessor(TextWriter trapFile) { - var path = NonGeneratedSourceLocation.TryAdjustRelativeMappedFilePath(Symbol.File.ValueText, Symbol.SyntaxTree.FilePath, Context.Extractor.Logger); + var path = Context.TryAdjustRelativeMappedFilePath(Symbol.File.ValueText, Symbol.SyntaxTree.FilePath); var file = File.Create(Context, path); trapFile.pragma_checksums(this, file, Symbol.Guid.ToString(), Symbol.Bytes.ToString()); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs index 281ce9e93491..a21b71ed1964 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs @@ -10,6 +10,7 @@ using Semmle.Util; using Semmle.Util.Logging; using Semmle.Extraction.CSharp.Populators; +using System.Reflection; namespace Semmle.Extraction.CSharp { @@ -38,14 +39,23 @@ public class Analyser : IDisposable public PathTransformer PathTransformer { get; } - protected Analyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer) + public IPathCache PathCache { get; } + + protected Analyser( + IProgressMonitor pm, + ILogger logger, + PathTransformer pathTransformer, + IPathCache pathCache, + bool addAssemblyTrapPrefix) { Logger = logger; + PathTransformer = pathTransformer; + PathCache = pathCache; this.addAssemblyTrapPrefix = addAssemblyTrapPrefix; + this.progressMonitor = pm; + Logger.Log(Severity.Info, "EXTRACTION STARTING at {0}", DateTime.Now); stopWatch.Start(); - progressMonitor = pm; - PathTransformer = pathTransformer; } /// @@ -333,11 +343,26 @@ public virtual void Dispose() /// /// Logs information about the extractor. /// - public void LogExtractorInfo(string extractorVersion) + public void LogExtractorInfo() { Logger.Log(Severity.Info, " Extractor: {0}", Environment.GetCommandLineArgs().First()); - Logger.Log(Severity.Info, " Extractor version: {0}", extractorVersion); + Logger.Log(Severity.Info, " Extractor version: {0}", Version); Logger.Log(Severity.Info, " Current working directory: {0}", Directory.GetCurrentDirectory()); } + + private static string Version + { + get + { + // the attribute for the git information are always attached to the entry assembly by our build system + var assembly = Assembly.GetEntryAssembly(); + var versionString = assembly?.GetCustomAttribute(); + if (versionString == null) + { + return "unknown (not built from internal bazel workspace)"; + } + return versionString.InformationalVersion; + } + } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs index e799eb4a387f..338b8490070a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.IO; using Microsoft.CodeAnalysis; using Semmle.Extraction.Entities; @@ -178,5 +179,25 @@ internal bool ExtractGenerics(CachedEntity entity) extractedGenerics.Add(entity.Label); return true; } + + public string TryAdjustRelativeMappedFilePath(string mappedToPath, string mappedFromPath) + { + if (!Path.IsPathRooted(mappedToPath)) + { + try + { + var fullPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(mappedFromPath)!, mappedToPath)); + Extractor.Logger.LogDebug($"Found relative path in line mapping: '{mappedToPath}', interpreting it as '{fullPath}'"); + + mappedToPath = fullPath; + } + catch (Exception e) + { + Extractor.Logger.LogDebug($"Failed to compute absolute path for relative path in line mapping: '{mappedToPath}': {e}"); + } + } + + return mappedToPath; + } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs index f6913103ad8c..cdeb915de58c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs @@ -105,7 +105,7 @@ public static ExitCode Run(string[] args) var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); var pathTransformer = new PathTransformer(canonicalPathCache); - using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, options.AssemblySensitiveTrap, pathTransformer); + using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap); try { @@ -144,7 +144,7 @@ public static ExitCode Run(string[] args) return ExitCode.Ok; } - return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, canonicalPathCache, stopwatch); + return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, stopwatch); } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { @@ -226,7 +226,7 @@ private static MetadataReference MakeReference(CommandLineReference reference, s /// The resolved references will be added (thread-safely) to the supplied /// list . /// - private static IEnumerable ResolveReferences(Microsoft.CodeAnalysis.CommandLineArguments args, Analyser analyser, CanonicalPathCache canonicalPathCache, BlockingCollection ret) + private static IEnumerable ResolveReferences(Microsoft.CodeAnalysis.CommandLineArguments args, Analyser analyser, BlockingCollection ret) { var referencePaths = new Lazy(() => FixedReferencePaths(args).ToArray()); return args.MetadataReferences.Select(clref => () => @@ -235,7 +235,7 @@ private static IEnumerable ResolveReferences(Microsoft.CodeAnalysis.Comm { if (File.Exists(clref.Reference)) { - var reference = MakeReference(clref, canonicalPathCache.GetCanonicalPath(clref.Reference)); + var reference = MakeReference(clref, analyser.PathCache.GetCanonicalPath(clref.Reference)); ret.Add(reference); } else @@ -252,7 +252,7 @@ private static IEnumerable ResolveReferences(Microsoft.CodeAnalysis.Comm var composed = referencePaths.Value .Select(path => Path.Combine(path, clref.Reference)) .Where(path => File.Exists(path)) - .Select(path => canonicalPathCache.GetCanonicalPath(path)) + .Select(path => analyser.PathCache.GetCanonicalPath(path)) .FirstOrDefault(); if (composed is not null) @@ -382,11 +382,10 @@ private static ExitCode AnalyseTracing( TracingAnalyser analyser, CSharpCommandLineArguments compilerArguments, Options options, - CanonicalPathCache canonicalPathCache, Stopwatch stopwatch) { return Analyse(stopwatch, analyser, options, - references => ResolveReferences(compilerArguments, analyser, canonicalPathCache, references), + references => ResolveReferences(compilerArguments, analyser, references), (analyser, syntaxTrees) => { var paths = compilerArguments.SourceFiles @@ -399,7 +398,7 @@ private static ExitCode AnalyseTracing( } return ReadSyntaxTrees( - paths.Select(canonicalPathCache.GetCanonicalPath), + paths.Select(analyser.PathCache.GetCanonicalPath), analyser, compilerArguments.ParseOptions, compilerArguments.Encoding, diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs index 82ea40c2d949..192fe7f34fd3 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs @@ -2,14 +2,15 @@ using System.Collections.Generic; using System.IO; using Microsoft.CodeAnalysis.CSharp; +using Semmle.Util; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp { public class StandaloneAnalyser : Analyser { - public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer) - : base(pm, logger, addAssemblyTrapPrefix, pathTransformer) + public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix) + : base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix) { } @@ -18,7 +19,7 @@ public void Initialize(string outputPath, IEnumerable<(string, string)> compilat compilation = compilationIn; extractor = new Extraction.Extractor(Directory.GetCurrentDirectory(), [], outputPath, compilationInfos, Logger, PathTransformer, ExtractorMode.Standalone, options.QlTest); this.options = options; - LogExtractorInfo(Extraction.Extractor.Version); + LogExtractorInfo(); SetReferencePaths(); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs index 43c5562736cd..5418ebed07f2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs @@ -13,8 +13,8 @@ public class TracingAnalyser : Analyser { private bool init; - public TracingAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer) - : base(pm, logger, addAssemblyTrapPrefix, pathTransformer) + public TracingAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix) + : base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix) { } @@ -25,7 +25,8 @@ public TracingAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrap /// A Boolean indicating whether to proceed with extraction. public bool BeginInitialize(IEnumerable roslynArgs) { - return init = LogRoslynArgs(roslynArgs, Extraction.Extractor.Version); + LogExtractorInfo(); + return init = LogRoslynArgs(roslynArgs); } /// @@ -47,11 +48,11 @@ public void EndInitialize( this.options = options; this.compilation = compilation; this.extractor = new Extraction.Extractor(cwd, args, GetOutputName(compilation, commandLineArguments), [], Logger, PathTransformer, ExtractorMode.None, options.QlTest); - LogDiagnostics(); + var errorCount = LogDiagnostics(compilation); SetReferencePaths(); - CompilationErrors += FilteredDiagnostics.Count(); + CompilationErrors += errorCount; } /// @@ -59,9 +60,8 @@ public void EndInitialize( /// /// The arguments passed to Roslyn. /// A Boolean indicating whether the same arguments have been logged previously. - private bool LogRoslynArgs(IEnumerable roslynArgs, string extractorVersion) + private bool LogRoslynArgs(IEnumerable roslynArgs) { - LogExtractorInfo(extractorVersion); Logger.Log(Severity.Info, $" Arguments to Roslyn: {string.Join(' ', roslynArgs)}"); var tempFile = Extractor.GetCSharpArgsLogPath(Path.GetRandomFileName()); @@ -137,27 +137,27 @@ private static string GetOutputName(CSharpCompilation compilation, return Path.Combine(commandLineArguments.OutputDirectory, commandLineArguments.OutputFileName); } -#nullable disable warnings - - /// - /// Logs detailed information about this invocation, - /// in the event that errors were detected. - /// - /// A Boolean indicating whether to proceed with extraction. - private void LogDiagnostics() + private int LogDiagnostics(CSharpCompilation compilation) { - foreach (var error in FilteredDiagnostics) + var filteredDiagnostics = compilation + .GetDiagnostics() + .Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id)) + .ToList(); + + foreach (var error in filteredDiagnostics) { Logger.Log(Severity.Error, " Compilation error: {0}", error); } - if (FilteredDiagnostics.Any()) + if (filteredDiagnostics.Count != 0) { foreach (var reference in compilation.References) { Logger.Log(Severity.Info, " Resolved reference {0}", reference.Display); } } + + return filteredDiagnostics.Count; } private static readonly HashSet errorsToIgnore = new HashSet @@ -166,17 +166,5 @@ private void LogDiagnostics() "CS1589", // XML referencing not supported "CS1569" // Error writing XML documentation }; - - private IEnumerable FilteredDiagnostics - { - get - { - return extractor is null || extractor.Mode.HasFlag(ExtractorMode.Standalone) || compilation is null ? Enumerable.Empty() : - compilation. - GetDiagnostics(). - Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id)); - } - } -#nullable restore warnings } } diff --git a/csharp/extractor/Semmle.Extraction/Context.cs b/csharp/extractor/Semmle.Extraction/Context.cs index 6697bfb06fdf..622d0a21ccd2 100644 --- a/csharp/extractor/Semmle.Extraction/Context.cs +++ b/csharp/extractor/Semmle.Extraction/Context.cs @@ -51,7 +51,7 @@ protected void DefineLabel(IEntity entity) try { writingLabel = true; - entity.DefineLabel(TrapWriter.Writer, Extractor); + entity.DefineLabel(TrapWriter.Writer); } finally { diff --git a/csharp/extractor/Semmle.Extraction/Entities/Base/Entity.cs b/csharp/extractor/Semmle.Extraction/Entities/Base/Entity.cs index 49186582b0de..e28b89ad0eb0 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/Base/Entity.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/Base/Entity.cs @@ -28,7 +28,7 @@ public virtual void WriteQuotedId(EscapingTextWriter trapFile) public abstract TrapStackBehaviour TrapStackBehaviour { get; } - public void DefineLabel(TextWriter trapFile, Extractor extractor) + public void DefineLabel(TextWriter trapFile) { trapFile.WriteLabel(this); trapFile.Write("="); @@ -40,7 +40,7 @@ public void DefineLabel(TextWriter trapFile, Extractor extractor) catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { trapFile.WriteLine("\""); - extractor.Message(new Message($"Unhandled exception generating id: {ex.Message}", ToString() ?? "", null, ex.StackTrace)); + Context.Extractor.Message(new Message($"Unhandled exception generating id: {ex.Message}", ToString() ?? "", null, ex.StackTrace)); } trapFile.WriteLine(); } diff --git a/csharp/extractor/Semmle.Extraction/Entities/Base/IEntity.cs b/csharp/extractor/Semmle.Extraction/Entities/Base/IEntity.cs index acb5d79d5971..3700e61b22ea 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/Base/IEntity.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/Base/IEntity.cs @@ -48,6 +48,6 @@ public interface IEntity /// TrapStackBehaviour TrapStackBehaviour { get; } - void DefineLabel(TextWriter trapFile, Extractor extractor); + void DefineLabel(TextWriter trapFile); } } diff --git a/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs b/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs index c8da1265559e..a57da8feabdf 100644 --- a/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using System.Reflection; using Semmle.Util.Logging; using CompilationInfo = (string key, string value); @@ -110,21 +109,6 @@ public void MissingNamespace(string fqdn, bool fromSource) public ILogger Logger { get; private set; } - public static string Version - { - get - { - // the attribute for the git information are always attached to the entry assembly by our build system - var assembly = Assembly.GetEntryAssembly(); - var versionString = assembly?.GetCustomAttribute(); - if (versionString == null) - { - return "unknown (not built from internal bazel workspace)"; - } - return versionString.InformationalVersion; - } - } - public PathTransformer PathTransformer { get; } } } From cdca6078284a6d3be9410e0cc97fe342aa8c1866 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 12 Jun 2024 11:43:12 +0200 Subject: [PATCH 3/6] Rename `Extractor` to `ExtractionContext` --- .../Entities/Assembly.cs | 8 ++++---- .../Entities/Compilations/Compilation.cs | 4 ++-- .../Compilations/CompilerDiagnostic.cs | 2 +- .../Entities/Expressions/Invocation.cs | 2 +- .../Semmle.Extraction.CSharp/Entities/File.cs | 2 +- .../Entities/OrdinaryMethod.cs | 2 +- .../Entities/Types/NamedType.cs | 2 +- .../Entities/UsingDirective.cs | 2 +- .../Extractor/Analyser.cs | 20 +++++++++---------- .../Extractor/Context.cs | 6 +++--- .../Extractor/StandaloneAnalyser.cs | 6 +++--- .../Extractor/TracingAnalyser.cs | 2 +- csharp/extractor/Semmle.Extraction/Context.cs | 12 +++++------ .../Semmle.Extraction/Entities/Base/Entity.cs | 2 +- .../Entities/ExtractionMessage.cs | 2 +- .../Semmle.Extraction/Entities/File.cs | 2 +- .../{Extractor.cs => ExtractionContext.cs} | 4 ++-- 17 files changed, 40 insertions(+), 40 deletions(-) rename csharp/extractor/Semmle.Extraction/Extractor/{Extractor.cs => ExtractionContext.cs} (92%) diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs index a826aa5e02c5..849051632531 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs @@ -17,7 +17,7 @@ private Assembly(Context cx, Microsoft.CodeAnalysis.Location? init) isOutputAssembly = init is null; if (isOutputAssembly) { - assemblyPath = cx.Extractor.OutputPath; + assemblyPath = cx.ExtractionContext.OutputPath; assembly = cx.Compilation.Assembly; } else @@ -25,7 +25,7 @@ private Assembly(Context cx, Microsoft.CodeAnalysis.Location? init) assembly = init!.MetadataModule!.ContainingAssembly; var identity = assembly.Identity; var idString = identity.Name + " " + identity.Version; - assemblyPath = cx.Extractor.GetAssemblyFile(idString); + assemblyPath = cx.ExtractionContext.GetAssemblyFile(idString); } } @@ -33,7 +33,7 @@ public override void Populate(TextWriter trapFile) { if (assemblyPath is not null) { - var isBuildlessOutputAssembly = isOutputAssembly && Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone); + var isBuildlessOutputAssembly = isOutputAssembly && Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone); var identifier = isBuildlessOutputAssembly ? "" : assembly.ToString() ?? ""; @@ -74,7 +74,7 @@ public static Assembly CreateOutputAssembly(Context cx) public override void WriteId(EscapingTextWriter trapFile) { - if (isOutputAssembly && Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone)) + if (isOutputAssembly && Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone)) { trapFile.Write("buildlessOutputAssembly"); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs index 505ab8f3ed68..0ba1965723eb 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs @@ -18,8 +18,8 @@ internal class Compilation : CachedEntity #nullable disable warnings private Compilation(Context cx) : base(cx, null) { - cwd = cx.Extractor.Cwd; - args = cx.Extractor.Args; + cwd = cx.ExtractionContext.Cwd; + args = cx.ExtractionContext.Args; hashCode = cwd.GetHashCode(); for (var i = 0; i < args.Length; i++) { diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs index c1227f2ffc0b..b76cbfb910cb 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs @@ -28,7 +28,7 @@ protected override void Populate(TextWriter trapFile) { if (messageCount == limit + 1) { - Context.Extractor.Logger.LogWarning($"Stopped logging {key} compiler diagnostics for the current compilation after reaching {limit}"); + Context.ExtractionContext.Logger.LogWarning($"Stopped logging {key} compiler diagnostics for the current compilation after reaching {limit}"); } return; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs index 5563f48f4f15..4db35fda985e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs @@ -133,7 +133,7 @@ public IMethodSymbol? TargetSymbol .Where(method => method.Parameters.Length >= Syntax.ArgumentList.Arguments.Count) .Where(method => method.Parameters.Count(p => !p.HasExplicitDefaultValue) <= Syntax.ArgumentList.Arguments.Count); - return Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) ? + return Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone) ? candidates.FirstOrDefault() : candidates.SingleOrDefault(); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs index c3239d378712..e0940191a314 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs @@ -61,7 +61,7 @@ public override void Populate(TextWriter trapFile) } } - trapFile.file_extraction_mode(this, Context.Extractor.Mode); + trapFile.file_extraction_mode(this, Context.ExtractionContext.Mode); } private bool IsPossiblyTextFile() diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs index c9ad1678ddd1..e4c84c3f7055 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs @@ -63,7 +63,7 @@ Symbol.ContainingType is INamedTypeSymbol nt && { if (method.MethodKind == MethodKind.ReducedExtension) { - cx.Extractor.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted."); + cx.ExtractionContext.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted."); } return OrdinaryMethodFactory.Instance.CreateEntityFromSymbol(cx, method); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs index 93a9add36c44..5b8eb20bb1a9 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs @@ -36,7 +36,7 @@ public override void Populate(TextWriter trapFile) if (Symbol.TypeKind == TypeKind.Error) { UnknownType.Create(Context); // make sure this exists so we can use it in `TypeRef::getReferencedType` - Context.Extractor.MissingType(Symbol.ToString()!, Context.FromSource); + Context.ExtractionContext.MissingType(Symbol.ToString()!, Context.FromSource); return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs index 294935abbf57..04fe80dbcde7 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs @@ -36,7 +36,7 @@ protected override void Populate(TextWriter trapFile) } else { - Context.Extractor.MissingNamespace(name.ToFullString(), Context.FromSource); + Context.ExtractionContext.MissingNamespace(name.ToFullString(), Context.FromSource); Context.ModelError(node, "Namespace not found"); return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs index a21b71ed1964..f82512f70c3e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs @@ -19,7 +19,7 @@ namespace Semmle.Extraction.CSharp /// public class Analyser : IDisposable { - protected Extraction.Extractor? extractor; + protected ExtractionContext? ExtractionContext; protected CSharpCompilation? compilation; protected CommonOptions? options; private protected Entities.Compilation? compilationEntity; @@ -108,12 +108,12 @@ protected void SetReferencePaths() var def = reader.GetAssemblyDefinition(); assemblyIdentity = reader.GetString(def.Name) + " " + def.Version; } - extractor.SetAssemblyFile(assemblyIdentity, refPath); + ExtractionContext.SetAssemblyFile(assemblyIdentity, refPath); } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { - extractor.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace)); + ExtractionContext.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace)); } } } @@ -158,7 +158,7 @@ private void DoAnalyseReferenceAssembly(PortableExecutableReference r) if (compilation.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly) { - var cx = new Context(extractor, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix); + var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix); foreach (var module in assembly.Modules) { @@ -201,7 +201,7 @@ private void DoExtractTree(SyntaxTree tree) if (!upToDate) { - var cx = new Context(extractor, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix); + var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix); // Ensure that the file itself is populated in case the source file is totally empty var root = tree.GetRoot(); Entities.File.Create(cx, root.SyntaxTree.FilePath); @@ -223,7 +223,7 @@ private void DoExtractTree(SyntaxTree tree) } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { - extractor.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace)); + ExtractionContext.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace)); } } @@ -231,7 +231,7 @@ private void DoAnalyseCompilation() { try { - var assemblyPath = extractor.OutputPath; + var assemblyPath = ExtractionContext.OutputPath; var stopwatch = new Stopwatch(); stopwatch.Start(); var currentTaskId = IncrementTaskCount(); @@ -241,11 +241,11 @@ private void DoAnalyseCompilation() var assembly = compilation.Assembly; var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false); compilationTrapFile = trapWriter; // Dispose later - var cx = new Context(extractor, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix); + var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix); compilationEntity = Entities.Compilation.Create(cx); - extractor.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value)); + ExtractionContext.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value)); ReportProgressTaskDone(currentTaskId, assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, AnalysisAction.Extracted); } @@ -328,7 +328,7 @@ public virtual void Dispose() /// /// Number of errors encountered during extraction. /// - private int ExtractorErrors => extractor?.Errors ?? 0; + private int ExtractorErrors => ExtractionContext?.Errors ?? 0; /// /// Number of errors encountered by the compiler. diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs index 338b8490070a..8a633b578b5f 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs @@ -77,7 +77,7 @@ internal void CacheLambdaParameterSymbol(IParameterSymbol param, SyntaxNode synt internal CommentProcessor CommentGenerator { get; } = new CommentProcessor(); - public Context(Extraction.Extractor e, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix) + public Context(ExtractionContext e, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix) : base(e, trapWriter, addAssemblyTrapPrefix) { Compilation = c; @@ -187,13 +187,13 @@ public string TryAdjustRelativeMappedFilePath(string mappedToPath, string mapped try { var fullPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(mappedFromPath)!, mappedToPath)); - Extractor.Logger.LogDebug($"Found relative path in line mapping: '{mappedToPath}', interpreting it as '{fullPath}'"); + ExtractionContext.Logger.LogDebug($"Found relative path in line mapping: '{mappedToPath}', interpreting it as '{fullPath}'"); mappedToPath = fullPath; } catch (Exception e) { - Extractor.Logger.LogDebug($"Failed to compute absolute path for relative path in line mapping: '{mappedToPath}': {e}"); + ExtractionContext.Logger.LogDebug($"Failed to compute absolute path for relative path in line mapping: '{mappedToPath}': {e}"); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs index 192fe7f34fd3..15d2eab00a4a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs @@ -17,7 +17,7 @@ public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer p public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options) { compilation = compilationIn; - extractor = new Extraction.Extractor(Directory.GetCurrentDirectory(), [], outputPath, compilationInfos, Logger, PathTransformer, ExtractorMode.Standalone, options.QlTest); + ExtractionContext = new ExtractionContext(Directory.GetCurrentDirectory(), [], outputPath, compilationInfos, Logger, PathTransformer, ExtractorMode.Standalone, options.QlTest); this.options = options; LogExtractorInfo(); SetReferencePaths(); @@ -25,9 +25,9 @@ public void Initialize(string outputPath, IEnumerable<(string, string)> compilat #nullable disable warnings - public IEnumerable MissingTypes => extractor.MissingTypes; + public IEnumerable MissingTypes => ExtractionContext.MissingTypes; - public IEnumerable MissingNamespaces => extractor.MissingNamespaces; + public IEnumerable MissingNamespaces => ExtractionContext.MissingNamespaces; #nullable restore warnings } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs index 5418ebed07f2..d349cf81dcbb 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs @@ -47,7 +47,7 @@ public void EndInitialize( throw new InternalError("EndInitialize called without BeginInitialize returning true"); this.options = options; this.compilation = compilation; - this.extractor = new Extraction.Extractor(cwd, args, GetOutputName(compilation, commandLineArguments), [], Logger, PathTransformer, ExtractorMode.None, options.QlTest); + this.ExtractionContext = new ExtractionContext(cwd, args, GetOutputName(compilation, commandLineArguments), [], Logger, PathTransformer, ExtractorMode.None, options.QlTest); var errorCount = LogDiagnostics(compilation); SetReferencePaths(); diff --git a/csharp/extractor/Semmle.Extraction/Context.cs b/csharp/extractor/Semmle.Extraction/Context.cs index 622d0a21ccd2..512ae874bf9a 100644 --- a/csharp/extractor/Semmle.Extraction/Context.cs +++ b/csharp/extractor/Semmle.Extraction/Context.cs @@ -18,7 +18,7 @@ public class Context /// /// Access various extraction functions, e.g. logger, trap writer. /// - public Extractor Extractor { get; } + public ExtractionContext ExtractionContext { get; } /// /// Access to the trap file. @@ -190,9 +190,9 @@ public void PopulateAll() } } - protected Context(Extractor extractor, TrapWriter trapWriter, bool shouldAddAssemblyTrapPrefix = false) + protected Context(ExtractionContext extractor, TrapWriter trapWriter, bool shouldAddAssemblyTrapPrefix = false) { - Extractor = extractor; + ExtractionContext = extractor; TrapWriter = trapWriter; ShouldAddAssemblyTrapPrefix = shouldAddAssemblyTrapPrefix; } @@ -274,7 +274,7 @@ private void Populate(ISymbol? optionalSymbol, CachedEntity entity) bool duplicationGuard, deferred; - if (Extractor.Mode is ExtractorMode.Standalone) + if (ExtractionContext.Mode is ExtractorMode.Standalone) { duplicationGuard = false; deferred = false; @@ -398,7 +398,7 @@ private void ExtractionError(string message, ISymbol? optionalSymbol, Entity opt private void ExtractionError(Message msg) { new ExtractionMessage(this, msg); - Extractor.Message(msg); + ExtractionContext.Message(msg); } private void ExtractionError(InternalError error) @@ -408,7 +408,7 @@ private void ExtractionError(InternalError error) private void ReportError(InternalError error) { - if (!Extractor.Mode.HasFlag(ExtractorMode.Standalone)) + if (!ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone)) throw error; ExtractionError(error); diff --git a/csharp/extractor/Semmle.Extraction/Entities/Base/Entity.cs b/csharp/extractor/Semmle.Extraction/Entities/Base/Entity.cs index e28b89ad0eb0..9cd645ae1d41 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/Base/Entity.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/Base/Entity.cs @@ -40,7 +40,7 @@ public void DefineLabel(TextWriter trapFile) catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { trapFile.WriteLine("\""); - Context.Extractor.Message(new Message($"Unhandled exception generating id: {ex.Message}", ToString() ?? "", null, ex.StackTrace)); + Context.ExtractionContext.Message(new Message($"Unhandled exception generating id: {ex.Message}", ToString() ?? "", null, ex.StackTrace)); } trapFile.WriteLine(); } diff --git a/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs b/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs index bc6ea5aa27dc..c27263e2cba8 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs @@ -25,7 +25,7 @@ protected override void Populate(TextWriter trapFile) { if (messageCount == limit + 1) { - Context.Extractor.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}"); + Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}"); } return; } diff --git a/csharp/extractor/Semmle.Extraction/Entities/File.cs b/csharp/extractor/Semmle.Extraction/Entities/File.cs index 952302360b1b..b703362feb8e 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/File.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/File.cs @@ -8,7 +8,7 @@ protected File(Context cx, string path) : base(cx, path) { originalPath = path; - transformedPathLazy = new Lazy(() => Context.Extractor.PathTransformer.Transform(originalPath)); + transformedPathLazy = new Lazy(() => Context.ExtractionContext.PathTransformer.Transform(originalPath)); } protected readonly string originalPath; diff --git a/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs b/csharp/extractor/Semmle.Extraction/Extractor/ExtractionContext.cs similarity index 92% rename from csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs rename to csharp/extractor/Semmle.Extraction/Extractor/ExtractionContext.cs index a57da8feabdf..3628e4bc80ea 100644 --- a/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction/Extractor/ExtractionContext.cs @@ -7,7 +7,7 @@ namespace Semmle.Extraction /// /// Implementation of the main extractor state. /// - public class Extractor + public class ExtractionContext { public string Cwd { get; init; } public string[] Args { get; init; } @@ -18,7 +18,7 @@ public class Extractor /// /// Creates a new extractor instance for one compilation unit. /// - public Extractor(string cwd, string[] args, string outputPath, IEnumerable compilationInfos, ILogger logger, PathTransformer pathTransformer, ExtractorMode mode, bool isQlTest) + public ExtractionContext(string cwd, string[] args, string outputPath, IEnumerable compilationInfos, ILogger logger, PathTransformer pathTransformer, ExtractorMode mode, bool isQlTest) { OutputPath = outputPath; Logger = logger; From 3551386a1ace60e4f1a8251f21ca65eaa72784d9 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 12 Jun 2024 12:30:08 +0200 Subject: [PATCH 4/6] Simplify standalone extractor --- .../Extractor.cs | 59 ++++++++----------- .../Extractor/Analyser.cs | 2 +- .../Extractor/Extractor.cs | 22 +++---- .../Extractor/StandaloneAnalyser.cs | 9 --- 4 files changed, 37 insertions(+), 55 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs index a72d04ac8ba4..adde7ea71115 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs @@ -42,17 +42,17 @@ private static void AnalyseStandalone( (compilation, options) => analyser.Initialize(output.FullName, extractionInput.CompilationInfos, compilation, options), () => { - foreach (var type in analyser.MissingNamespaces) + foreach (var type in analyser.ExtractionContext!.MissingNamespaces) { progressMonitor.MissingNamespace(type); } - foreach (var type in analyser.MissingTypes) + foreach (var type in analyser.ExtractionContext!.MissingTypes) { progressMonitor.MissingType(type); } - progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count()); + progressMonitor.MissingSummary(analyser.ExtractionContext!.MissingTypes.Count(), analyser.ExtractionContext!.MissingNamespaces.Count()); }); } finally @@ -69,29 +69,6 @@ private static void AnalyseStandalone( } } - private static void ExtractStandalone( - ExtractionInput extractionInput, - IProgressMonitor pm, - ILogger logger, - CommonOptions options) - { - var stopwatch = new Stopwatch(); - stopwatch.Start(); - - var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); - var pathTransformer = new PathTransformer(canonicalPathCache); - - using var analyser = new StandaloneAnalyser(pm, logger, pathTransformer, canonicalPathCache, false); - try - { - AnalyseStandalone(analyser, extractionInput, options, pm, stopwatch); - } - catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] - { - analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", ex); - } - } - private class ExtractionProgress : IProgressMonitor { public ExtractionProgress(ILogger output) @@ -141,8 +118,8 @@ public record ExtractionInput(IEnumerable Sources, IEnumerable R public static ExitCode Run(Options options) { - var stopwatch = new Stopwatch(); - stopwatch.Start(); + var overallStopwatch = new Stopwatch(); + overallStopwatch.Start(); using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true); logger.Log(Severity.Info, "Extracting C# with build-mode set to 'none'"); @@ -158,12 +135,26 @@ public static ExitCode Run(Options options) logger.Log(Severity.Info, ""); logger.Log(Severity.Info, "Extracting..."); - ExtractStandalone( - new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos), - new ExtractionProgress(logger), - fileLogger, - options); - logger.Log(Severity.Info, $"Extraction completed in {stopwatch.Elapsed}"); + + var analyzerStopwatch = new Stopwatch(); + analyzerStopwatch.Start(); + + var canonicalPathCache = CanonicalPathCache.Create(fileLogger, 1000); + var pathTransformer = new PathTransformer(canonicalPathCache); + + var progressMonitor = new ExtractionProgress(logger); + using var analyser = new StandaloneAnalyser(progressMonitor, fileLogger, pathTransformer, canonicalPathCache, false); + try + { + var extractionInput = new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos); + AnalyseStandalone(analyser, extractionInput, options, progressMonitor, analyzerStopwatch); + } + catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] + { + fileLogger.Log(Severity.Error, " Unhandled exception: {0}", ex); + } + + logger.Log(Severity.Info, $"Extraction completed in {overallStopwatch.Elapsed}"); return ExitCode.Ok; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs index f82512f70c3e..a57c269128ec 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs @@ -19,7 +19,7 @@ namespace Semmle.Extraction.CSharp /// public class Analyser : IDisposable { - protected ExtractionContext? ExtractionContext; + public ExtractionContext? ExtractionContext { get; protected set; } protected CSharpCompilation? compilation; protected CommonOptions? options; private protected Entities.Compilation? compilationEntity; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs index cdeb915de58c..700718fc1e55 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs @@ -93,20 +93,13 @@ public static ILogger MakeLogger(Verbosity verbosity, bool includeConsole) /// public static ExitCode Run(string[] args) { - var stopwatch = new Stopwatch(); - stopwatch.Start(); + var analyzerStopwatch = new Stopwatch(); + analyzerStopwatch.Start(); var options = Options.CreateWithEnvironment(args); - var workingDirectory = Directory.GetCurrentDirectory(); - var compilerArgs = options.CompilerArguments.ToArray(); using var logger = MakeLogger(options.Verbosity, options.Console); - var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); - var pathTransformer = new PathTransformer(canonicalPathCache); - - using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap); - try { if (options.ProjectsToLoad.Any()) @@ -115,13 +108,20 @@ public static ExitCode Run(string[] args) } var compilerVersion = new CompilerVersion(options); - if (compilerVersion.SkipExtraction) { logger.Log(Severity.Warning, " Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason); return ExitCode.Ok; } + var workingDirectory = Directory.GetCurrentDirectory(); + var compilerArgs = options.CompilerArguments.ToArray(); + + var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); + var pathTransformer = new PathTransformer(canonicalPathCache); + + using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap); + var compilerArguments = CSharpCommandLineParser.Default.Parse( compilerVersion.ArgsWithResponse, workingDirectory, @@ -144,7 +144,7 @@ public static ExitCode Run(string[] args) return ExitCode.Ok; } - return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, stopwatch); + return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, analyzerStopwatch); } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs index 15d2eab00a4a..3ce315b372ff 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs @@ -1,4 +1,3 @@ -using System; using System.Collections.Generic; using System.IO; using Microsoft.CodeAnalysis.CSharp; @@ -22,13 +21,5 @@ public void Initialize(string outputPath, IEnumerable<(string, string)> compilat LogExtractorInfo(); SetReferencePaths(); } - -#nullable disable warnings - - public IEnumerable MissingTypes => ExtractionContext.MissingTypes; - - public IEnumerable MissingNamespaces => ExtractionContext.MissingNamespaces; - -#nullable restore warnings } } From e3cbf1479a57f09084f73773a44b9fd8c9130189 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 12 Jun 2024 13:25:34 +0200 Subject: [PATCH 5/6] Fix quality issue --- .../Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs index d349cf81dcbb..417b45c37bb1 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs @@ -48,7 +48,7 @@ public void EndInitialize( this.options = options; this.compilation = compilation; this.ExtractionContext = new ExtractionContext(cwd, args, GetOutputName(compilation, commandLineArguments), [], Logger, PathTransformer, ExtractorMode.None, options.QlTest); - var errorCount = LogDiagnostics(compilation); + var errorCount = LogDiagnostics(); SetReferencePaths(); @@ -137,9 +137,9 @@ private static string GetOutputName(CSharpCompilation compilation, return Path.Combine(commandLineArguments.OutputDirectory, commandLineArguments.OutputFileName); } - private int LogDiagnostics(CSharpCompilation compilation) + private int LogDiagnostics() { - var filteredDiagnostics = compilation + var filteredDiagnostics = compilation! .GetDiagnostics() .Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id)) .ToList(); From 0df6a1c4b18877572a126cf3e1d4b7756b5288ad Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 12 Jun 2024 14:48:16 +0200 Subject: [PATCH 6/6] Fix quality issues --- .../extractor/Semmle.Extraction.CSharp/Extractor/Context.cs | 4 ++-- csharp/extractor/Semmle.Extraction/Context.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs index 8a633b578b5f..57e9a5ca9f17 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs @@ -77,8 +77,8 @@ internal void CacheLambdaParameterSymbol(IParameterSymbol param, SyntaxNode synt internal CommentProcessor CommentGenerator { get; } = new CommentProcessor(); - public Context(ExtractionContext e, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix) - : base(e, trapWriter, addAssemblyTrapPrefix) + public Context(ExtractionContext extractionContext, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix) + : base(extractionContext, trapWriter, addAssemblyTrapPrefix) { Compilation = c; this.scope = scope; diff --git a/csharp/extractor/Semmle.Extraction/Context.cs b/csharp/extractor/Semmle.Extraction/Context.cs index 512ae874bf9a..c83a79fbddc3 100644 --- a/csharp/extractor/Semmle.Extraction/Context.cs +++ b/csharp/extractor/Semmle.Extraction/Context.cs @@ -190,9 +190,9 @@ public void PopulateAll() } } - protected Context(ExtractionContext extractor, TrapWriter trapWriter, bool shouldAddAssemblyTrapPrefix = false) + protected Context(ExtractionContext extractionContext, TrapWriter trapWriter, bool shouldAddAssemblyTrapPrefix = false) { - ExtractionContext = extractor; + ExtractionContext = extractionContext; TrapWriter = trapWriter; ShouldAddAssemblyTrapPrefix = shouldAddAssemblyTrapPrefix; }