From 05500568c9f214176d87878df5a33cdc5a431bbb Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Tue, 13 Aug 2024 16:22:19 +0200 Subject: [PATCH 1/2] C#: Minor adjustment to limiting the number of extractor messages --- .../Entities/Compilations/CompilerDiagnostic.cs | 1 - .../Semmle.Extraction/Entities/ExtractionMessage.cs | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs index b76cbfb910cb..b4c0cdca7300 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs @@ -21,7 +21,6 @@ public CompilerDiagnostic(Context cx, Microsoft.CodeAnalysis.Diagnostic diag, Co protected override void Populate(TextWriter trapFile) { - // The below doesn't limit the extractor messages to the exact limit, but it's good enough. var key = diagnostic.Id; var messageCount = compilation.messageCounts.AddOrUpdate(key, 1, (_, c) => c + 1); if (messageCount > limit) diff --git a/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs b/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs index c27263e2cba8..41535bcefbb4 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs @@ -19,11 +19,10 @@ public ExtractionMessage(Context cx, Message msg) : base(cx) protected override void Populate(TextWriter trapFile) { - // The below doesn't limit the extractor messages to the exact limit, but it's good enough. - Interlocked.Increment(ref messageCount); - if (messageCount > limit) + var val = Interlocked.Increment(ref messageCount); + if (val > limit) { - if (messageCount == limit + 1) + if (val == limit + 1) { Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}"); } From 5f74ead99b4ca7d81a7e0ba2004824b340222768 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Tue, 13 Aug 2024 17:24:43 +0200 Subject: [PATCH 2/2] C#: Store extractor message in DB for limit reached --- .../Entities/ExtractionMessage.cs | 24 +++++++++++++------ csharp/extractor/Semmle.Extraction/Tuples.cs | 4 ++-- .../all-platforms/standalone/Diag.expected | 4 +++- .../all-platforms/standalone/Diag.ql | 4 ++++ 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs b/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs index 41535bcefbb4..f417a170c10b 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs @@ -10,26 +10,36 @@ internal class ExtractionMessage : FreshEntity private static int messageCount = 0; private readonly Message msg; + private readonly bool bypassLimit; - public ExtractionMessage(Context cx, Message msg) : base(cx) + public ExtractionMessage(Context cx, Message msg) : this(cx, msg, bypassLimit: false) { + } + + private ExtractionMessage(Context cx, Message msg, bool bypassLimit) : base(cx) + { + this.bypassLimit = bypassLimit; this.msg = msg; TryPopulate(); } protected override void Populate(TextWriter trapFile) { - var val = Interlocked.Increment(ref messageCount); - if (val > limit) + if (!bypassLimit) { - if (val == limit + 1) + var val = Interlocked.Increment(ref messageCount); + if (val > limit) { - Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}"); + if (val == limit + 1) + { + Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}"); + _ = new ExtractionMessage(Context, new Message($"Stopped logging extractor messages after reaching {limit}", null, null, null, Util.Logging.Severity.Warning), bypassLimit: true); + } + return; } - return; } - trapFile.extractor_messages(this, msg.Severity, "C# extractor", msg.Text, msg.EntityText ?? string.Empty, + trapFile.extractor_messages(this, msg.Severity, msg.Text, msg.EntityText ?? string.Empty, msg.Location ?? Context.CreateLocation(), msg.StackTrace ?? string.Empty); } } diff --git a/csharp/extractor/Semmle.Extraction/Tuples.cs b/csharp/extractor/Semmle.Extraction/Tuples.cs index 81a3149325f3..cddec9322863 100644 --- a/csharp/extractor/Semmle.Extraction/Tuples.cs +++ b/csharp/extractor/Semmle.Extraction/Tuples.cs @@ -12,9 +12,9 @@ public static void containerparent(this System.IO.TextWriter trapFile, Folder pa trapFile.WriteTuple("containerparent", parent, child); } - internal static void extractor_messages(this System.IO.TextWriter trapFile, ExtractionMessage error, Semmle.Util.Logging.Severity severity, string origin, string errorMessage, string entityText, Location location, string stackTrace) + internal static void extractor_messages(this System.IO.TextWriter trapFile, ExtractionMessage error, Semmle.Util.Logging.Severity severity, string errorMessage, string entityText, Location location, string stackTrace) { - trapFile.WriteTuple("extractor_messages", error, (int)severity, origin, errorMessage, entityText, location, stackTrace); + trapFile.WriteTuple("extractor_messages", error, (int)severity, "C# extractor", errorMessage, entityText, location, stackTrace); } public static void files(this System.IO.TextWriter trapFile, File file, string fullName) diff --git a/csharp/ql/integration-tests/all-platforms/standalone/Diag.expected b/csharp/ql/integration-tests/all-platforms/standalone/Diag.expected index b48630869ee8..6d84e27e5cea 100644 --- a/csharp/ql/integration-tests/all-platforms/standalone/Diag.expected +++ b/csharp/ql/integration-tests/all-platforms/standalone/Diag.expected @@ -1,7 +1,9 @@ extractorMessages -| 5 | +| 6 | compilerDiagnostics | 4 | +extractorMessagesLeachedLimit +| Program.cs:1:1:1:0 | Stopped logging extractor messages after reaching 5 | compilationInfo | Compiler diagnostic count for CS0103 | 3.0 | | Compiler diagnostic count for CS8019 | 7.0 | diff --git a/csharp/ql/integration-tests/all-platforms/standalone/Diag.ql b/csharp/ql/integration-tests/all-platforms/standalone/Diag.ql index e391b345b20b..8b1fbae6b2fd 100644 --- a/csharp/ql/integration-tests/all-platforms/standalone/Diag.ql +++ b/csharp/ql/integration-tests/all-platforms/standalone/Diag.ql @@ -5,6 +5,10 @@ query predicate extractorMessages(int c) { c = count(ExtractorMessage msg) } query predicate compilerDiagnostics(int c) { c = count(Diagnostic diag) } +query predicate extractorMessagesLeachedLimit(ExtractorMessage msg) { + msg.getText().indexOf("Stopped logging") = 0 +} + query predicate compilationInfo(string key, float value) { exists(Compilation c, string infoValue | infoValue = c.getInfo(key) and key.matches("Compiler diagnostic count for%")