Merge pull request #16732 from tamasvajk/refactor/extraction-states

C#: Refactor extractor state classes and simplify extraction code

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -42,17 +42,17 @@ private static void AnalyseStandalone(
                     (compilation, options) => analyser.Initialize(output.FullName, extractionInput.CompilationInfos, compilation, options),
                     () =>
                     {
-                        foreach (var type in analyser.MissingNamespaces)
+                        foreach (var type in analyser.ExtractionContext!.MissingNamespaces)
                         {
                             progressMonitor.MissingNamespace(type);
                         }
 
-                        foreach (var type in analyser.MissingTypes)
+                        foreach (var type in analyser.ExtractionContext!.MissingTypes)
                         {
                             progressMonitor.MissingType(type);
                         }
 
-                        progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
+                        progressMonitor.MissingSummary(analyser.ExtractionContext!.MissingTypes.Count(), analyser.ExtractionContext!.MissingNamespaces.Count());
                     });
             }
             finally
@@ -69,29 +69,6 @@ private static void AnalyseStandalone(
             }
         }
 
-        private static void ExtractStandalone(
-            ExtractionInput extractionInput,
-            IProgressMonitor pm,
-            ILogger logger,
-            CommonOptions options)
-        {
-            var stopwatch = new Stopwatch();
-            stopwatch.Start();
-
-            var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
-            var pathTransformer = new PathTransformer(canonicalPathCache);
-
-            using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
-            try
-            {
-                AnalyseStandalone(analyser, extractionInput, options, pm, stopwatch);
-            }
-            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
-            {
-                analyser.Logger.Log(Severity.Error, "  Unhandled exception: {0}", ex);
-            }
-        }
-
         private class ExtractionProgress : IProgressMonitor
         {
             public ExtractionProgress(ILogger output)
@@ -141,8 +118,8 @@ public record ExtractionInput(IEnumerable<string> Sources, IEnumerable<string> R
 
         public static ExitCode Run(Options options)
         {
-            var stopwatch = new Stopwatch();
-            stopwatch.Start();
+            var overallStopwatch = new Stopwatch();
+            overallStopwatch.Start();
 
             using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
             logger.Log(Severity.Info, "Extracting C# with build-mode set to 'none'");
@@ -158,12 +135,26 @@ public static ExitCode Run(Options options)
 
             logger.Log(Severity.Info, "");
             logger.Log(Severity.Info, "Extracting...");
-            ExtractStandalone(
-                new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos),
-                new ExtractionProgress(logger),
-                fileLogger,
-                options);
-            logger.Log(Severity.Info, $"Extraction completed in {stopwatch.Elapsed}");
+
+            var analyzerStopwatch = new Stopwatch();
+            analyzerStopwatch.Start();
+
+            var canonicalPathCache = CanonicalPathCache.Create(fileLogger, 1000);
+            var pathTransformer = new PathTransformer(canonicalPathCache);
+
+            var progressMonitor = new ExtractionProgress(logger);
+            using var analyser = new StandaloneAnalyser(progressMonitor, fileLogger, pathTransformer, canonicalPathCache, false);
+            try
+            {
+                var extractionInput = new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos);
+                AnalyseStandalone(analyser, extractionInput, options, progressMonitor, analyzerStopwatch);
+            }
+            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+            {
+                fileLogger.Log(Severity.Error, "  Unhandled exception: {0}", ex);
+            }
+
+            logger.Log(Severity.Info, $"Extraction completed in {overallStopwatch.Elapsed}");
 
             return ExitCode.Ok;
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs

@@ -17,23 +17,23 @@ private Assembly(Context cx, Microsoft.CodeAnalysis.Location? init)
             isOutputAssembly = init is null;
             if (isOutputAssembly)
             {
-                assemblyPath = cx.Extractor.OutputPath;
+                assemblyPath = cx.ExtractionContext.OutputPath;
                 assembly = cx.Compilation.Assembly;
             }
             else
             {
                 assembly = init!.MetadataModule!.ContainingAssembly;
                 var identity = assembly.Identity;
                 var idString = identity.Name + " " + identity.Version;
-                assemblyPath = cx.Extractor.GetAssemblyFile(idString);
+                assemblyPath = cx.ExtractionContext.GetAssemblyFile(idString);
             }
         }
 
         public override void Populate(TextWriter trapFile)
         {
             if (assemblyPath is not null)
             {
-                var isBuildlessOutputAssembly = isOutputAssembly && Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone);
+                var isBuildlessOutputAssembly = isOutputAssembly && Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone);
                 var identifier = isBuildlessOutputAssembly
                     ? ""
                     : assembly.ToString() ?? "";
@@ -74,7 +74,7 @@ public static Assembly CreateOutputAssembly(Context cx)
 
         public override void WriteId(EscapingTextWriter trapFile)
         {
-            if (isOutputAssembly && Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
+            if (isOutputAssembly && Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone))
             {
                 trapFile.Write("buildlessOutputAssembly");
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

@@ -18,8 +18,8 @@ internal class Compilation : CachedEntity<object>
 #nullable disable warnings
         private Compilation(Context cx) : base(cx, null)
         {
-            cwd = cx.Extractor.Cwd;
-            args = cx.Extractor.Args;
+            cwd = cx.ExtractionContext.Cwd;
+            args = cx.ExtractionContext.Args;
             hashCode = cwd.GetHashCode();
             for (var i = 0; i < args.Length; i++)
             {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs

@@ -28,7 +28,7 @@ protected override void Populate(TextWriter trapFile)
             {
                 if (messageCount == limit + 1)
                 {
-                    Context.Extractor.Logger.LogWarning($"Stopped logging {key} compiler diagnostics for the current compilation after reaching {limit}");
+                    Context.ExtractionContext.Logger.LogWarning($"Stopped logging {key} compiler diagnostics for the current compilation after reaching {limit}");
                 }
 
                 return;

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs

@@ -133,7 +133,7 @@ public IMethodSymbol? TargetSymbol
                         .Where(method => method.Parameters.Length >= Syntax.ArgumentList.Arguments.Count)
                         .Where(method => method.Parameters.Count(p => !p.HasExplicitDefaultValue) <= Syntax.ArgumentList.Arguments.Count);
 
-                    return Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) ?
+                    return Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone) ?
                         candidates.FirstOrDefault() :
                         candidates.SingleOrDefault();
                 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs

@@ -61,7 +61,7 @@ public override void Populate(TextWriter trapFile)
                 }
             }
 
-            trapFile.file_extraction_mode(this, Context.Extractor.Mode);
+            trapFile.file_extraction_mode(this, Context.ExtractionContext.Mode);
         }
 
         private bool IsPossiblyTextFile()

csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs

@@ -27,7 +27,7 @@ public override void Populate(TextWriter trapFile)
             var mapped = Symbol.GetMappedLineSpan();
             if (mapped.HasMappedPath && mapped.IsValid)
             {
-                var path = TryAdjustRelativeMappedFilePath(mapped.Path, Position.Path, Context.Extractor.Logger);
+                var path = Context.TryAdjustRelativeMappedFilePath(mapped.Path, Position.Path);
                 var mappedLoc = Create(Context, Location.Create(path, default, mapped.Span));
 
                 trapFile.locations_mapped(this, mappedLoc);
@@ -64,25 +64,5 @@ private class SourceLocationFactory : CachedEntityFactory<Location, NonGenerated
 
             public override NonGeneratedSourceLocation Create(Context cx, Location init) => new NonGeneratedSourceLocation(cx, init);
         }
-
-        public static string TryAdjustRelativeMappedFilePath(string mappedToPath, string mappedFromPath, ILogger logger)
-        {
-            if (!Path.IsPathRooted(mappedToPath))
-            {
-                try
-                {
-                    var fullPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(mappedFromPath)!, mappedToPath));
-                    logger.LogDebug($"Found relative path in line mapping: '{mappedToPath}', interpreting it as '{fullPath}'");
-
-                    mappedToPath = fullPath;
-                }
-                catch (Exception e)
-                {
-                    logger.LogDebug($"Failed to compute absolute path for relative path in line mapping: '{mappedToPath}': {e}");
-                }
-            }
-
-            return mappedToPath;
-        }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs

@@ -63,7 +63,7 @@ Symbol.ContainingType is INamedTypeSymbol nt &&
         {
             if (method.MethodKind == MethodKind.ReducedExtension)
             {
-                cx.Extractor.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
+                cx.ExtractionContext.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
             }
 
             return OrdinaryMethodFactory.Instance.CreateEntityFromSymbol(cx, method);

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineOrSpanDirective.cs

@@ -28,7 +28,7 @@ protected override void PopulatePreprocessor(TextWriter trapFile)
             var path = Symbol.File.ValueText;
             if (!string.IsNullOrWhiteSpace(path))
             {
-                path = NonGeneratedSourceLocation.TryAdjustRelativeMappedFilePath(path, Symbol.SyntaxTree.FilePath, Context.Extractor.Logger);
+                path = Context.TryAdjustRelativeMappedFilePath(path, Symbol.SyntaxTree.FilePath);
                 var file = File.Create(Context, path);
                 trapFile.directive_line_file(this, file);
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs

@@ -12,7 +12,7 @@ private PragmaChecksumDirective(Context cx, PragmaChecksumDirectiveTriviaSyntax
 
         protected override void PopulatePreprocessor(TextWriter trapFile)
         {
-            var path = NonGeneratedSourceLocation.TryAdjustRelativeMappedFilePath(Symbol.File.ValueText, Symbol.SyntaxTree.FilePath, Context.Extractor.Logger);
+            var path = Context.TryAdjustRelativeMappedFilePath(Symbol.File.ValueText, Symbol.SyntaxTree.FilePath);
             var file = File.Create(Context, path);
             trapFile.pragma_checksums(this, file, Symbol.Guid.ToString(), Symbol.Bytes.ToString());
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs

@@ -36,7 +36,7 @@ public override void Populate(TextWriter trapFile)
             if (Symbol.TypeKind == TypeKind.Error)
             {
                 UnknownType.Create(Context); // make sure this exists so we can use it in `TypeRef::getReferencedType`
-                Context.Extractor.MissingType(Symbol.ToString()!, Context.FromSource);
+                Context.ExtractionContext.MissingType(Symbol.ToString()!, Context.FromSource);
                 return;
             }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs

@@ -36,7 +36,7 @@ protected override void Populate(TextWriter trapFile)
                 }
                 else
                 {
-                    Context.Extractor.MissingNamespace(name.ToFullString(), Context.FromSource);
+                    Context.ExtractionContext.MissingNamespace(name.ToFullString(), Context.FromSource);
                     Context.ModelError(node, "Namespace not found");
                     return;
                 }

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -10,6 +10,7 @@
 using Semmle.Util;
 using Semmle.Util.Logging;
 using Semmle.Extraction.CSharp.Populators;
+using System.Reflection;
 
 namespace Semmle.Extraction.CSharp
 {
@@ -18,7 +19,7 @@ namespace Semmle.Extraction.CSharp
     /// </summary>
     public class Analyser : IDisposable
     {
-        protected Extraction.Extractor? extractor;
+        public ExtractionContext? ExtractionContext { get; protected set; }
         protected CSharpCompilation? compilation;
         protected CommonOptions? options;
         private protected Entities.Compilation? compilationEntity;
@@ -38,14 +39,23 @@ public class Analyser : IDisposable
 
         public PathTransformer PathTransformer { get; }
 
-        protected Analyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
+        public IPathCache PathCache { get; }
+
+        protected Analyser(
+            IProgressMonitor pm,
+            ILogger logger,
+            PathTransformer pathTransformer,
+            IPathCache pathCache,
+            bool addAssemblyTrapPrefix)
         {
             Logger = logger;
+            PathTransformer = pathTransformer;
+            PathCache = pathCache;
             this.addAssemblyTrapPrefix = addAssemblyTrapPrefix;
+            this.progressMonitor = pm;
+
             Logger.Log(Severity.Info, "EXTRACTION STARTING at {0}", DateTime.Now);
             stopWatch.Start();
-            progressMonitor = pm;
-            PathTransformer = pathTransformer;
         }
 
         /// <summary>
@@ -98,12 +108,12 @@ protected void SetReferencePaths()
                         var def = reader.GetAssemblyDefinition();
                         assemblyIdentity = reader.GetString(def.Name) + " " + def.Version;
                     }
-                    extractor.SetAssemblyFile(assemblyIdentity, refPath);
+                    ExtractionContext.SetAssemblyFile(assemblyIdentity, refPath);
 
                 }
                 catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
                 {
-                    extractor.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace));
+                    ExtractionContext.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace));
                 }
             }
         }
@@ -148,7 +158,7 @@ private void DoAnalyseReferenceAssembly(PortableExecutableReference r)
 
                     if (compilation.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly)
                     {
-                        var cx = new Context(extractor, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
+                        var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
 
                         foreach (var module in assembly.Modules)
                         {
@@ -191,7 +201,7 @@ private void DoExtractTree(SyntaxTree tree)
 
                 if (!upToDate)
                 {
-                    var cx = new Context(extractor, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
+                    var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
                     // Ensure that the file itself is populated in case the source file is totally empty
                     var root = tree.GetRoot();
                     Entities.File.Create(cx, root.SyntaxTree.FilePath);
@@ -213,15 +223,15 @@ private void DoExtractTree(SyntaxTree tree)
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
-                extractor.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace));
+                ExtractionContext.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace));
             }
         }
 
         private void DoAnalyseCompilation()
         {
             try
             {
-                var assemblyPath = extractor.OutputPath;
+                var assemblyPath = ExtractionContext.OutputPath;
                 var stopwatch = new Stopwatch();
                 stopwatch.Start();
                 var currentTaskId = IncrementTaskCount();
@@ -231,11 +241,11 @@ private void DoAnalyseCompilation()
                 var assembly = compilation.Assembly;
                 var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
                 compilationTrapFile = trapWriter;  // Dispose later
-                var cx = new Context(extractor, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
+                var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
 
                 compilationEntity = Entities.Compilation.Create(cx);
 
-                extractor.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
+                ExtractionContext.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
 
                 ReportProgressTaskDone(currentTaskId, assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, AnalysisAction.Extracted);
             }
@@ -318,7 +328,7 @@ public virtual void Dispose()
         /// <summary>
         /// Number of errors encountered during extraction.
         /// </summary>
-        private int ExtractorErrors => extractor?.Errors ?? 0;
+        private int ExtractorErrors => ExtractionContext?.Errors ?? 0;
 
         /// <summary>
         /// Number of errors encountered by the compiler.
@@ -333,11 +343,26 @@ public virtual void Dispose()
         /// <summary>
         /// Logs information about the extractor.
         /// </summary>
-        public void LogExtractorInfo(string extractorVersion)
+        public void LogExtractorInfo()
         {
             Logger.Log(Severity.Info, "  Extractor: {0}", Environment.GetCommandLineArgs().First());
-            Logger.Log(Severity.Info, "  Extractor version: {0}", extractorVersion);
+            Logger.Log(Severity.Info, "  Extractor version: {0}", Version);
             Logger.Log(Severity.Info, "  Current working directory: {0}", Directory.GetCurrentDirectory());
         }
+
+        private static string Version
+        {
+            get
+            {
+                // the attribute for the git information are always attached to the entry assembly by our build system
+                var assembly = Assembly.GetEntryAssembly();
+                var versionString = assembly?.GetCustomAttribute<AssemblyInformationalVersionAttribute>();
+                if (versionString == null)
+                {
+                    return "unknown (not built from internal bazel workspace)";
+                }
+                return versionString.InformationalVersion;
+            }
+        }
     }
 }