diff --git a/csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel b/csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel index 47e8ec48fd4c..42495f63ed44 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel +++ b/csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel @@ -6,7 +6,6 @@ load( codeql_csharp_library( name = "Semmle.Extraction.CSharp", srcs = glob([ - "_Base/**/*.cs", "CodeAnalysisExtensions/**/*.cs", "Comments/**/*.cs", "Entities/**/*.cs", diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntity`1.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntity.cs similarity index 92% rename from csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntity`1.cs rename to csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntity.cs index 988f813c5198..ed0a21bc0836 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntity`1.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntity.cs @@ -2,14 +2,14 @@ using System.IO; using Microsoft.CodeAnalysis; -namespace Semmle.Extraction +namespace Semmle.Extraction.CSharp.Entities { /// /// A cached entity. /// /// The property is used as label in caching. /// - public abstract class CachedEntity : CSharp.LabelledEntity + public abstract class CachedEntity : LabelledEntity { protected CachedEntity(Context context) : base(context) { @@ -62,7 +62,7 @@ public override bool Equals(object? obj) return other?.GetType() == GetType() && Equals(other.Symbol, Symbol); } - public override CSharp.TrapStackBehaviour TrapStackBehaviour => CSharp.TrapStackBehaviour.NoLabel; + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; } /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntityFactory.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntityFactory.cs index 68cb8a393b5e..6753dce9c722 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntityFactory.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntityFactory.cs @@ -3,7 +3,7 @@ namespace Semmle.Extraction.CSharp /// /// A factory for creating cached entities. /// - public abstract class CachedEntityFactory where TEntity : CachedEntity + public abstract class CachedEntityFactory where TEntity : Entities.CachedEntity { public abstract TEntity Create(Context cx, TInit init); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntityFactoryExtensions.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntityFactoryExtensions.cs index 1101db458666..d7ca1293e33c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntityFactoryExtensions.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntityFactoryExtensions.cs @@ -16,7 +16,7 @@ public static class CachedEntityFactoryExtensions /// The initializer for the entity. /// The entity. public static TEntity CreateEntity(this CachedEntityFactory factory, Context cx, object cacheKey, TInit init) - where TEntity : CachedEntity => cx.CreateEntity(factory, cacheKey, init); + where TEntity : Entities.CachedEntity => cx.CreateEntity(factory, cacheKey, init); /// /// Creates and populates a new entity from an `ISymbol`, or returns the existing one @@ -30,6 +30,6 @@ public static TEntity CreateEntity(this CachedEntityFactoryThe entity. public static TEntity CreateEntityFromSymbol(this CachedEntityFactory factory, Context cx, TSymbol init) where TSymbol : ISymbol - where TEntity : CachedEntity => cx.CreateEntityFromSymbol(factory, init); + where TEntity : Entities.CachedEntity => cx.CreateEntityFromSymbol(factory, init); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedSymbol.cs similarity index 98% rename from csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs rename to csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedSymbol.cs index ad5640b90e65..c39eb6076b57 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedSymbol.cs @@ -1,8 +1,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Reflection.Metadata; -using System.Reflection.Metadata.Ecma335; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp.Syntax; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/Entity.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/Entity.cs index c699c965d931..16e656f3fba2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/Entity.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/Entity.cs @@ -6,9 +6,9 @@ namespace Semmle.Extraction.CSharp { public abstract class Entity : IEntity { - public virtual Semmle.Extraction.Context Context { get; } + public virtual Context Context { get; } - protected Entity(Semmle.Extraction.Context context) + protected Entity(Context context) { this.Context = context; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/FreshEntity.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/FreshEntity.cs index e4690e940c98..87776fa8c6b1 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/FreshEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/FreshEntity.cs @@ -1,11 +1,11 @@ using System.IO; -namespace Semmle.Extraction +namespace Semmle.Extraction.CSharp.Entities { /// /// An entity which has a default "*" ID assigned to it. /// - public abstract class FreshEntity : CSharp.UnlabelledEntity + public abstract class FreshEntity : UnlabelledEntity { protected FreshEntity(Context cx) : base(cx) { @@ -33,6 +33,6 @@ public string DebugContents public override Microsoft.CodeAnalysis.Location? ReportingLocation => null; - public override CSharp.TrapStackBehaviour TrapStackBehaviour => CSharp.TrapStackBehaviour.NoLabel; + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/LabelledEntity.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/LabelledEntity.cs index e664d82b2b8f..e458c935b0ee 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/LabelledEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/LabelledEntity.cs @@ -2,7 +2,7 @@ namespace Semmle.Extraction.CSharp { public abstract class LabelledEntity : Entity { - protected LabelledEntity(Semmle.Extraction.Context cx) : base(cx) + protected LabelledEntity(Context cx) : base(cx) { } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/UnlabelledEntity.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/UnlabelledEntity.cs index 61d3dc8527bf..2bfcc35b274c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/UnlabelledEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/UnlabelledEntity.cs @@ -2,7 +2,7 @@ namespace Semmle.Extraction.CSharp { public abstract class UnlabelledEntity : Entity { - protected UnlabelledEntity(Extraction.Context cx) : base(cx) + protected UnlabelledEntity(Context cx) : base(cx) { cx.AddFreshLabel(this); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedEntity.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedEntity.cs deleted file mode 100644 index 149ed2e142ab..000000000000 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedEntity.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Semmle.Extraction.CSharp.Entities -{ - public abstract class CachedEntity : Extraction.CachedEntity where T : notnull - { - public override Context Context => (Context)base.Context; - - protected CachedEntity(Context context, T symbol) - : base(context, symbol) - { - } - } -} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs index 9029eb071d44..7638eefce125 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs @@ -1,5 +1,4 @@ using System.IO; -using Semmle.Extraction.Entities; namespace Semmle.Extraction.CSharp.Entities { diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs index ecb5d51d44d5..2c74775460da 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs @@ -3,7 +3,6 @@ using System.IO; using System.Linq; using Microsoft.CodeAnalysis; -using Semmle.Extraction.Entities; using Semmle.Util; namespace Semmle.Extraction.CSharp.Entities diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/ExtractionMessage.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/ExtractionMessage.cs index e3a85037d285..f8a771ae6d0e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/ExtractionMessage.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/ExtractionMessage.cs @@ -2,9 +2,8 @@ using System.IO; using System.Threading; using Semmle.Util; -using Semmle.Extraction.CSharp; -namespace Semmle.Extraction.Entities +namespace Semmle.Extraction.CSharp.Entities { internal class ExtractionMessage : FreshEntity { @@ -41,7 +40,7 @@ protected override void Populate(TextWriter trapFile) if (val == limit + 1) { Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}"); - _ = new ExtractionMessage(Context, new Message($"Stopped logging extractor messages after reaching {limit}", null, null, null, Util.Logging.Severity.Warning), bypassLimit: true); + _ = new ExtractionMessage(Context, new Message($"Stopped logging extractor messages after reaching {limit}", null, null, null, Semmle.Util.Logging.Severity.Warning), bypassLimit: true); } return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs index d95c0944f0fb..0a91eb57ecdf 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs @@ -4,7 +4,6 @@ using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp.Syntax; using Semmle.Extraction.CSharp.Entities.Expressions; -using Semmle.Extraction.Entities; using Semmle.Extraction.Kinds; namespace Semmle.Extraction.CSharp.Entities diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/FreshEntity.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/FreshEntity.cs deleted file mode 100644 index 79144e109d41..000000000000 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/FreshEntity.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Semmle.Extraction.CSharp.Entities -{ - internal abstract class FreshEntity : Extraction.FreshEntity - { - public override Context Context => (Context)base.Context; - - protected FreshEntity(Context cx) - : base(cx) - { - } - } -} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Locations/File.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Locations/File.cs index 05b339edf013..f4b9d99af5bc 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Locations/File.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Locations/File.cs @@ -6,7 +6,7 @@ namespace Semmle.Extraction.CSharp.Entities { - public class File : Extraction.CachedEntity + public class File : CachedEntity { protected readonly string originalPath; private readonly Lazy transformedPathLazy; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Locations/Location.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Locations/Location.cs index 0e67eac507b9..9f9e15e33f33 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Locations/Location.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Locations/Location.cs @@ -1,7 +1,7 @@ namespace Semmle.Extraction.CSharp.Entities { #nullable disable warnings - public abstract class Location : Extraction.CachedEntity + public abstract class Location : CachedEntity { #nullable restore warnings protected Location(Context cx, Microsoft.CodeAnalysis.Location? init) diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs index aa68593d7d3c..cb1bb6e930da 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs @@ -3,7 +3,7 @@ namespace Semmle.Extraction.CSharp.Entities { - internal class Modifier : Extraction.CachedEntity + internal class Modifier : CachedEntity { private Modifier(Context cx, string init) : base(cx, init) { } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs index 8c6a65ad5538..76d518776caa 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs @@ -4,7 +4,6 @@ using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp.Syntax; using Semmle.Extraction.CSharp.Populators; -using Semmle.Extraction.Entities; namespace Semmle.Extraction.CSharp.Entities { diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statement`1.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statement`1.cs index 0d7bdd31354b..3b16cfc31f0f 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statement`1.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statement`1.cs @@ -1,7 +1,6 @@ using System.IO; using Microsoft.CodeAnalysis.CSharp; using Semmle.Extraction.CSharp.Populators; -using Semmle.Extraction.Entities; namespace Semmle.Extraction.CSharp.Entities { diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/SyntheticEmptyBlock.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/SyntheticEmptyBlock.cs index 670d338d2b3c..6d6ebba24fb9 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/SyntheticEmptyBlock.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/SyntheticEmptyBlock.cs @@ -1,7 +1,6 @@ using System.IO; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; -using Semmle.Extraction.Entities; using Semmle.Extraction.Kinds; namespace Semmle.Extraction.CSharp.Entities.Statements diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs index 4146cc810594..8d819d715f9f 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs @@ -2,8 +2,9 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; +using System.Linq; using Microsoft.CodeAnalysis; -using Semmle.Extraction.Entities; +using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp { @@ -11,8 +12,453 @@ namespace Semmle.Extraction.CSharp /// State that needs to be available throughout the extraction process. /// There is one Context object per trap output file. /// - public class Context : Extraction.Context + public class Context { + /// + /// Access various extraction functions, e.g. logger, trap writer. + /// + public ExtractionContext ExtractionContext { get; } + + /// + /// Access to the trap file. + /// + public TrapWriter TrapWriter { get; } + + /// + /// Holds if assembly information should be prefixed to TRAP labels. + /// + public bool ShouldAddAssemblyTrapPrefix { get; } + + public IList TrapStackSuffix { get; } = new List(); + + private int GetNewId() => TrapWriter.IdCounter++; + + // A recursion guard against writing to the trap file whilst writing an id to the trap file. + private bool writingLabel = false; + + private readonly Queue labelQueue = []; + + protected void DefineLabel(IEntity entity) + { + if (writingLabel) + { + // Don't define a label whilst writing a label. + labelQueue.Enqueue(entity); + } + else + { + try + { + writingLabel = true; + entity.DefineLabel(TrapWriter.Writer); + } + finally + { + writingLabel = false; + if (labelQueue.Any()) + { + DefineLabel(labelQueue.Dequeue()); + } + } + } + } + +#if DEBUG_LABELS + private void CheckEntityHasUniqueLabel(string id, CachedEntity entity) + { + if (idLabelCache.ContainsKey(id)) + { + this.Extractor.Message(new Message("Label collision for " + id, entity.Label.ToString(), CreateLocation(entity.ReportingLocation), "", Severity.Warning)); + } + else + { + idLabelCache[id] = entity; + } + } +#endif + + protected Label GetNewLabel() => new Label(GetNewId()); + + internal TEntity CreateEntity(CachedEntityFactory factory, object cacheKey, TInit init) + where TEntity : Entities.CachedEntity => + cacheKey is ISymbol s ? CreateEntity(factory, s, init, symbolEntityCache) : CreateEntity(factory, cacheKey, init, objectEntityCache); + + internal TEntity CreateEntityFromSymbol(CachedEntityFactory factory, TSymbol init) + where TSymbol : ISymbol + where TEntity : Entities.CachedEntity => CreateEntity(factory, init, init, symbolEntityCache); + + + /// + /// Creates and populates a new entity, or returns the existing one from the cache. + /// + /// The entity factory. + /// The key used for caching. + /// The initializer for the entity. + /// The dictionary to use for caching. + /// The new/existing entity. + private TEntity CreateEntity(CachedEntityFactory factory, TCacheKey cacheKey, TInit init, IDictionary dictionary) + where TCacheKey : notnull + where TEntity : Entities.CachedEntity + { + if (dictionary.TryGetValue(cacheKey, out var cached)) + return (TEntity)cached; + + using (StackGuard) + { + var label = GetNewLabel(); + var entity = factory.Create(this, init); + entity.Label = label; + + dictionary[cacheKey] = entity; + + DefineLabel(entity); + if (entity.NeedsPopulation) + Populate(init as ISymbol, entity); + +#if DEBUG_LABELS + using var id = new EscapingTextWriter(); + entity.WriteQuotedId(id); + CheckEntityHasUniqueLabel(id.ToString(), entity); +#endif + + return entity; + } + } + + /// + /// Creates a fresh label with ID "*", and set it on the + /// supplied object. + /// + internal void AddFreshLabel(Entity entity) + { + entity.Label = GetNewLabel(); + entity.DefineFreshLabel(TrapWriter.Writer); + } + +#if DEBUG_LABELS + private readonly Dictionary idLabelCache = new Dictionary(); +#endif + + private readonly IDictionary objectEntityCache = new Dictionary(); + private readonly IDictionary symbolEntityCache = new Dictionary(10000, SymbolEqualityComparer.Default); + + /// + /// Queue of items to populate later. + /// The only reason for this is so that the call stack does not + /// grow indefinitely, causing a potential stack overflow. + /// + private readonly Queue populateQueue = new Queue(); + + /// + /// Enqueue the given action to be performed later. + /// + /// The action to run. + public void PopulateLater(Action a, bool preserveDuplicationKey = true) + { + var key = preserveDuplicationKey ? GetCurrentTagStackKey() : null; + if (key is not null) + { + // If we are currently executing with a duplication guard, then the same + // guard must be used for the deferred action + populateQueue.Enqueue(() => WithDuplicationGuard(key, a)); + } + else + { + populateQueue.Enqueue(a); + } + } + + /// + /// Runs the main populate loop until there's nothing left to populate. + /// + public void PopulateAll() + { + while (populateQueue.Any()) + { + try + { + populateQueue.Dequeue()(); + } + catch (InternalError ex) + { + ExtractionError(new Message(ex.Text, ex.EntityText, CreateLocation(ex.Location), ex.StackTrace)); + } + catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] + { + ExtractionError($"Uncaught exception. {ex.Message}", null, CreateLocation(), ex.StackTrace); + } + } + } + + private int currentRecursiveDepth = 0; + private const int maxRecursiveDepth = 150; + + private void EnterScope() + { + if (currentRecursiveDepth >= maxRecursiveDepth) + throw new StackOverflowException($"Maximum nesting depth of {maxRecursiveDepth} exceeded"); + ++currentRecursiveDepth; + } + + private void ExitScope() + { + --currentRecursiveDepth; + } + + public IDisposable StackGuard => new ScopeGuard(this); + + private sealed class ScopeGuard : IDisposable + { + private readonly Context cx; + + public ScopeGuard(Context c) + { + cx = c; + cx.EnterScope(); + } + + public void Dispose() + { + cx.ExitScope(); + } + } + + private class PushEmitter : ITrapEmitter + { + private readonly Key key; + + public PushEmitter(Key key) + { + this.key = key; + } + + public void EmitTrap(TextWriter trapFile) + { + trapFile.Write(".push "); + key.AppendTo(trapFile); + trapFile.WriteLine(); + } + } + + private class PopEmitter : ITrapEmitter + { + public void EmitTrap(TextWriter trapFile) + { + trapFile.WriteLine(".pop"); + } + } + + private readonly Stack tagStack = new Stack(); + + /// + /// Populates an entity, handling the tag stack appropriately + /// + /// Symbol for reporting errors. + /// The entity to populate. + /// Thrown on invalid trap stack behaviour. + private void Populate(ISymbol? optionalSymbol, Entities.CachedEntity entity) + { + if (writingLabel) + { + // Don't write tuples etc if we're currently defining a label + PopulateLater(() => Populate(optionalSymbol, entity)); + return; + } + + bool duplicationGuard, deferred; + + if (ExtractionContext.Mode is ExtractorMode.Standalone) + { + duplicationGuard = false; + deferred = false; + } + else + { + switch (entity.TrapStackBehaviour) + { + case TrapStackBehaviour.NeedsLabel: + if (!tagStack.Any()) + ExtractionError("TagStack unexpectedly empty", optionalSymbol, entity); + duplicationGuard = false; + deferred = false; + break; + case TrapStackBehaviour.NoLabel: + duplicationGuard = false; + deferred = tagStack.Any(); + break; + case TrapStackBehaviour.OptionalLabel: + duplicationGuard = false; + deferred = false; + break; + case TrapStackBehaviour.PushesLabel: + duplicationGuard = true; + deferred = tagStack.Any(); + break; + default: + throw new InternalError("Unexpected TrapStackBehaviour"); + } + } + + var a = duplicationGuard && IsEntityDuplicationGuarded(entity, out var loc) + ? (() => + { + var args = new object[TrapStackSuffix.Count + 2]; + args[0] = entity; + args[1] = loc; + for (var i = 0; i < TrapStackSuffix.Count; i++) + { + args[i + 2] = TrapStackSuffix[i]; + } + WithDuplicationGuard(new Key(args), () => entity.Populate(TrapWriter.Writer)); + }) + : (Action)(() => this.Try(null, optionalSymbol, () => entity.Populate(TrapWriter.Writer))); + + if (deferred) + populateQueue.Enqueue(a); + else + a(); + } + + protected Key? GetCurrentTagStackKey() => tagStack.Count > 0 + ? tagStack.Peek() + : null; + + /// + /// Log an extraction error. + /// + /// The error message. + /// A textual representation of the failed entity. + /// The location of the error. + /// An optional stack trace of the error, or null. + /// The severity of the error. + public void ExtractionError(string message, string? entityText, Entities.Location? location, string? stackTrace = null, Severity severity = Severity.Error) + { + var msg = new Message(message, entityText, location, stackTrace, severity); + ExtractionError(msg); + } + + /// + /// Log an extraction error. + /// + /// The text of the message. + /// The symbol of the error, or null. + /// The entity of the error, or null. + private void ExtractionError(string message, ISymbol? optionalSymbol, Entity optionalEntity) + { + if (!(optionalSymbol is null)) + { + ExtractionError(message, optionalSymbol.ToDisplayString(), CreateLocation(optionalSymbol.Locations.BestOrDefault())); + } + else if (!(optionalEntity is null)) + { + ExtractionError(message, optionalEntity.Label.ToString(), CreateLocation(optionalEntity.ReportingLocation)); + } + else + { + ExtractionError(message, null, CreateLocation()); + } + } + + /// + /// Log an extraction message. + /// + /// The message to log. + private void ExtractionError(Message msg) + { + _ = new Entities.ExtractionMessage(this, msg); + ExtractionContext.Message(msg); + } + + private void ExtractionError(InternalError error) + { + ExtractionError(new Message(error.Message, error.EntityText, CreateLocation(error.Location), error.StackTrace, Severity.Error)); + } + + private void ReportError(InternalError error) + { + if (!ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone)) + throw error; + + ExtractionError(error); + } + + /// + /// Signal an error in the program model. + /// + /// The syntax node causing the failure. + /// The error message. + public void ModelError(SyntaxNode node, string msg) + { + ReportError(new InternalError(node, msg)); + } + + /// + /// Signal an error in the program model. + /// + /// Symbol causing the error. + /// The error message. + public void ModelError(ISymbol symbol, string msg) + { + ReportError(new InternalError(symbol, msg)); + } + + /// + /// Signal an error in the program model. + /// + /// The location of the error. + /// The error message. + public void ModelError(CSharp.Entities.Location loc, string msg) + { + ReportError(new InternalError(loc.ReportingLocation, msg)); + } + + /// + /// Signal an error in the program model. + /// + /// The error message. + public void ModelError(string msg) + { + ReportError(new InternalError(msg)); + } + + /// + /// Tries the supplied action , and logs an uncaught + /// exception error if the action fails. + /// + /// Optional syntax node for error reporting. + /// Optional symbol for error reporting. + /// The action to perform. + public void Try(SyntaxNode? node, ISymbol? symbol, Action a) + { + try + { + a(); + } + catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] + { + Message message; + + if (node is not null) + { + message = Message.Create(this, ex.Message, node, ex.StackTrace); + } + else if (symbol is not null) + { + message = Message.Create(this, ex.Message, symbol, ex.StackTrace); + } + else if (ex is InternalError ie) + { + message = new Message(ie.Text, ie.EntityText, CreateLocation(ie.Location), ex.StackTrace); + } + else + { + message = new Message($"Uncaught exception. {ex.Message}", null, CreateLocation(), ex.StackTrace); + } + + ExtractionError(message); + } + } + /// /// The program database provided by Roslyn. /// There's one per syntax tree, which makes things awkward. @@ -77,9 +523,11 @@ internal void CacheLambdaParameterSymbol(IParameterSymbol param, SyntaxNode synt internal CommentProcessor CommentGenerator { get; } = new CommentProcessor(); - public Context(ExtractionContext extractionContext, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix) - : base(extractionContext, trapWriter, addAssemblyTrapPrefix) + public Context(ExtractionContext extractionContext, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool shouldAddAssemblyTrapPrefix = false) { + ExtractionContext = extractionContext; + TrapWriter = trapWriter; + ShouldAddAssemblyTrapPrefix = shouldAddAssemblyTrapPrefix; Compilation = c; this.scope = scope; } @@ -102,7 +550,11 @@ public bool Defines(ISymbol symbol) => !SymbolEqualityComparer.Default.Equals(symbol, symbol.OriginalDefinition) || scope.InScope(symbol); - public override void WithDuplicationGuard(Key key, Action a) + /// + /// Runs the given action , guarding for trap duplication + /// based on key . + /// + public void WithDuplicationGuard(Key key, Action a) { if (IsAssemblyScope) { @@ -113,18 +565,28 @@ public override void WithDuplicationGuard(Key key, Action a) } else { - base.WithDuplicationGuard(key, a); + tagStack.Push(key); + TrapWriter.Emit(new PushEmitter(key)); + try + { + a(); + } + finally + { + TrapWriter.Emit(new PopEmitter()); + tagStack.Pop(); + } } } - public override Entities.Location CreateLocation() + public Entities.Location CreateLocation() { return SourceTree is null ? Entities.GeneratedLocation.Create(this) : CreateLocation(Microsoft.CodeAnalysis.Location.Create(SourceTree, Microsoft.CodeAnalysis.Text.TextSpan.FromBounds(0, 0))); } - public override Entities.Location CreateLocation(Microsoft.CodeAnalysis.Location? location) + public Entities.Location CreateLocation(Microsoft.CodeAnalysis.Location? location) { return (location is null || location.Kind == LocationKind.None) ? Entities.GeneratedLocation.Create(this) @@ -145,7 +607,7 @@ public void BindComments(Entity entity, Microsoft.CodeAnalysis.Location? l) CommentGenerator.AddElement(entity.Label, duplicationGuardKey, l); } - protected override bool IsEntityDuplicationGuarded(IEntity entity, [NotNullWhen(true)] out Entities.Location? loc) + private bool IsEntityDuplicationGuarded(IEntity entity, [NotNullWhen(true)] out Entities.Location? loc) { if (CreateLocation(entity.ReportingLocation) is Entities.NonGeneratedSourceLocation l) { @@ -169,7 +631,7 @@ protected override bool IsEntityDuplicationGuarded(IEntity entity, [NotNullWhen( /// /// The entity to extract. /// True only on the first call for a particular entity. - internal bool ExtractGenerics(CachedEntity entity) + internal bool ExtractGenerics(CSharp.Entities.CachedEntity entity) { if (extractedGenerics.Contains(entity.Label)) { diff --git a/csharp/extractor/Semmle.Extraction.CSharp/_Base/Message.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Message.cs similarity index 73% rename from csharp/extractor/Semmle.Extraction.CSharp/_Base/Message.cs rename to csharp/extractor/Semmle.Extraction.CSharp/Extractor/Message.cs index e99255be6bf9..4825c6c3fe1b 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/_Base/Message.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Message.cs @@ -13,9 +13,9 @@ public class Message public string Text { get; } public string? StackTrace { get; } public string? EntityText { get; } - public CSharp.Entities.Location? Location { get; } + public Entities.Location? Location { get; } - public Message(string text, string? entityText, CSharp.Entities.Location? location, string? stackTrace = null, Severity severity = Severity.Error) + public Message(string text, string? entityText, Entities.Location? location, string? stackTrace = null, Severity severity = Severity.Error) { Severity = severity; Text = text; @@ -24,12 +24,12 @@ public Message(string text, string? entityText, CSharp.Entities.Location? locati Location = location; } - public static Message Create(Extraction.Context cx, string text, ISymbol symbol, string? stackTrace = null, Severity severity = Severity.Error) + public static Message Create(Context cx, string text, ISymbol symbol, string? stackTrace = null, Severity severity = Severity.Error) { return new Message(text, symbol.ToString(), cx.CreateLocation(symbol.Locations.BestOrDefault()), stackTrace, severity); } - public static Message Create(Extraction.Context cx, string text, SyntaxNode node, string? stackTrace = null, Severity severity = Severity.Error) + public static Message Create(Context cx, string text, SyntaxNode node, string? stackTrace = null, Severity severity = Severity.Error) { return new Message(text, node.ToString(), cx.CreateLocation(node.GetLocation()), stackTrace, severity); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Trap/Tuples.cs b/csharp/extractor/Semmle.Extraction.CSharp/Trap/Tuples.cs index 092728d3be45..8960b6adb67f 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Trap/Tuples.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Trap/Tuples.cs @@ -3,7 +3,6 @@ using Semmle.Util; using Semmle.Extraction.CSharp.Entities; using Semmle.Extraction.CSharp.Entities.Expressions; -using Semmle.Extraction.Entities; using Semmle.Extraction.Kinds; namespace Semmle.Extraction.CSharp diff --git a/csharp/extractor/Semmle.Extraction.CSharp/_Base/Context.cs b/csharp/extractor/Semmle.Extraction.CSharp/_Base/Context.cs deleted file mode 100644 index 9d574f3d8b50..000000000000 --- a/csharp/extractor/Semmle.Extraction.CSharp/_Base/Context.cs +++ /dev/null @@ -1,501 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; -using Microsoft.CodeAnalysis; -using Semmle.Util.Logging; -using Semmle.Extraction.Entities; -using Semmle.Extraction.CSharp; - -namespace Semmle.Extraction -{ - /// - /// State that needs to be available throughout the extraction process. - /// There is one Context object per trap output file. - /// - public class Context - { - /// - /// Access various extraction functions, e.g. logger, trap writer. - /// - public ExtractionContext ExtractionContext { get; } - - /// - /// Access to the trap file. - /// - public TrapWriter TrapWriter { get; } - - /// - /// Holds if assembly information should be prefixed to TRAP labels. - /// - public bool ShouldAddAssemblyTrapPrefix { get; } - - public IList TrapStackSuffix { get; } = new List(); - - private int GetNewId() => TrapWriter.IdCounter++; - - // A recursion guard against writing to the trap file whilst writing an id to the trap file. - private bool writingLabel = false; - - private readonly Queue labelQueue = []; - - protected void DefineLabel(IEntity entity) - { - if (writingLabel) - { - // Don't define a label whilst writing a label. - labelQueue.Enqueue(entity); - } - else - { - try - { - writingLabel = true; - entity.DefineLabel(TrapWriter.Writer); - } - finally - { - writingLabel = false; - if (labelQueue.Any()) - { - DefineLabel(labelQueue.Dequeue()); - } - } - } - } - -#if DEBUG_LABELS - private void CheckEntityHasUniqueLabel(string id, CachedEntity entity) - { - if (idLabelCache.ContainsKey(id)) - { - this.Extractor.Message(new Message("Label collision for " + id, entity.Label.ToString(), CreateLocation(entity.ReportingLocation), "", Severity.Warning)); - } - else - { - idLabelCache[id] = entity; - } - } -#endif - - protected Label GetNewLabel() => new Label(GetNewId()); - - internal TEntity CreateEntity(CachedEntityFactory factory, object cacheKey, TInit init) - where TEntity : CachedEntity => - cacheKey is ISymbol s ? CreateEntity(factory, s, init, symbolEntityCache) : CreateEntity(factory, cacheKey, init, objectEntityCache); - - internal TEntity CreateEntityFromSymbol(CachedEntityFactory factory, TSymbol init) - where TSymbol : ISymbol - where TEntity : CachedEntity => CreateEntity(factory, init, init, symbolEntityCache); - - /// - /// Creates and populates a new entity, or returns the existing one from the cache. - /// - /// The entity factory. - /// The key used for caching. - /// The initializer for the entity. - /// The dictionary to use for caching. - /// The new/existing entity. - private TEntity CreateEntity(CachedEntityFactory factory, TCacheKey cacheKey, TInit init, IDictionary dictionary) - where TCacheKey : notnull - where TEntity : CachedEntity - { - if (dictionary.TryGetValue(cacheKey, out var cached)) - return (TEntity)cached; - - using (StackGuard) - { - var label = GetNewLabel(); - var entity = factory.Create((CSharp.Context)this, init); - entity.Label = label; - - dictionary[cacheKey] = entity; - - DefineLabel(entity); - if (entity.NeedsPopulation) - Populate(init as ISymbol, entity); - -#if DEBUG_LABELS - using var id = new EscapingTextWriter(); - entity.WriteQuotedId(id); - CheckEntityHasUniqueLabel(id.ToString(), entity); -#endif - - return entity; - } - } - - /// - /// Creates a fresh label with ID "*", and set it on the - /// supplied object. - /// - internal void AddFreshLabel(Entity entity) - { - entity.Label = GetNewLabel(); - entity.DefineFreshLabel(TrapWriter.Writer); - } - -#if DEBUG_LABELS - private readonly Dictionary idLabelCache = new Dictionary(); -#endif - - private readonly IDictionary objectEntityCache = new Dictionary(); - private readonly IDictionary symbolEntityCache = new Dictionary(10000, SymbolEqualityComparer.Default); - - /// - /// Queue of items to populate later. - /// The only reason for this is so that the call stack does not - /// grow indefinitely, causing a potential stack overflow. - /// - private readonly Queue populateQueue = new Queue(); - - /// - /// Enqueue the given action to be performed later. - /// - /// The action to run. - public void PopulateLater(Action a, bool preserveDuplicationKey = true) - { - var key = preserveDuplicationKey ? GetCurrentTagStackKey() : null; - if (key is not null) - { - // If we are currently executing with a duplication guard, then the same - // guard must be used for the deferred action - populateQueue.Enqueue(() => WithDuplicationGuard(key, a)); - } - else - { - populateQueue.Enqueue(a); - } - } - - /// - /// Runs the main populate loop until there's nothing left to populate. - /// - public void PopulateAll() - { - while (populateQueue.Any()) - { - try - { - populateQueue.Dequeue()(); - } - catch (InternalError ex) - { - ExtractionError(new Message(ex.Text, ex.EntityText, CreateLocation(ex.Location), ex.StackTrace)); - } - catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] - { - ExtractionError($"Uncaught exception. {ex.Message}", null, CreateLocation(), ex.StackTrace); - } - } - } - - protected Context(ExtractionContext extractionContext, TrapWriter trapWriter, bool shouldAddAssemblyTrapPrefix = false) - { - ExtractionContext = extractionContext; - TrapWriter = trapWriter; - ShouldAddAssemblyTrapPrefix = shouldAddAssemblyTrapPrefix; - } - - private int currentRecursiveDepth = 0; - private const int maxRecursiveDepth = 150; - - private void EnterScope() - { - if (currentRecursiveDepth >= maxRecursiveDepth) - throw new StackOverflowException($"Maximum nesting depth of {maxRecursiveDepth} exceeded"); - ++currentRecursiveDepth; - } - - private void ExitScope() - { - --currentRecursiveDepth; - } - - public IDisposable StackGuard => new ScopeGuard(this); - - private sealed class ScopeGuard : IDisposable - { - private readonly Context cx; - - public ScopeGuard(Context c) - { - cx = c; - cx.EnterScope(); - } - - public void Dispose() - { - cx.ExitScope(); - } - } - - private class PushEmitter : ITrapEmitter - { - private readonly Key key; - - public PushEmitter(Key key) - { - this.key = key; - } - - public void EmitTrap(TextWriter trapFile) - { - trapFile.Write(".push "); - key.AppendTo(trapFile); - trapFile.WriteLine(); - } - } - - private class PopEmitter : ITrapEmitter - { - public void EmitTrap(TextWriter trapFile) - { - trapFile.WriteLine(".pop"); - } - } - - private readonly Stack tagStack = new Stack(); - - /// - /// Populates an entity, handling the tag stack appropriately - /// - /// Symbol for reporting errors. - /// The entity to populate. - /// Thrown on invalid trap stack behaviour. - private void Populate(ISymbol? optionalSymbol, CachedEntity entity) - { - if (writingLabel) - { - // Don't write tuples etc if we're currently defining a label - PopulateLater(() => Populate(optionalSymbol, entity)); - return; - } - - bool duplicationGuard, deferred; - - if (ExtractionContext.Mode is ExtractorMode.Standalone) - { - duplicationGuard = false; - deferred = false; - } - else - { - switch (entity.TrapStackBehaviour) - { - case TrapStackBehaviour.NeedsLabel: - if (!tagStack.Any()) - ExtractionError("TagStack unexpectedly empty", optionalSymbol, entity); - duplicationGuard = false; - deferred = false; - break; - case TrapStackBehaviour.NoLabel: - duplicationGuard = false; - deferred = tagStack.Any(); - break; - case TrapStackBehaviour.OptionalLabel: - duplicationGuard = false; - deferred = false; - break; - case TrapStackBehaviour.PushesLabel: - duplicationGuard = true; - deferred = duplicationGuard && tagStack.Any(); - break; - default: - throw new InternalError("Unexpected TrapStackBehaviour"); - } - } - - var a = duplicationGuard && IsEntityDuplicationGuarded(entity, out var loc) - ? (() => - { - var args = new object[TrapStackSuffix.Count + 2]; - args[0] = entity; - args[1] = loc; - for (var i = 0; i < TrapStackSuffix.Count; i++) - { - args[i + 2] = TrapStackSuffix[i]; - } - WithDuplicationGuard(new Key(args), () => entity.Populate(TrapWriter.Writer)); - }) - : (Action)(() => this.Try(null, optionalSymbol, () => entity.Populate(TrapWriter.Writer))); - - if (deferred) - populateQueue.Enqueue(a); - else - a(); - } - - protected virtual bool IsEntityDuplicationGuarded(IEntity entity, [NotNullWhen(returnValue: true)] out CSharp.Entities.Location? loc) - { - loc = null; - return false; - } - - /// - /// Runs the given action , guarding for trap duplication - /// based on key . - /// - public virtual void WithDuplicationGuard(Key key, Action a) - { - tagStack.Push(key); - TrapWriter.Emit(new PushEmitter(key)); - try - { - a(); - } - finally - { - TrapWriter.Emit(new PopEmitter()); - tagStack.Pop(); - } - } - - protected Key? GetCurrentTagStackKey() => tagStack.Count > 0 - ? tagStack.Peek() - : null; - - /// - /// Log an extraction error. - /// - /// The error message. - /// A textual representation of the failed entity. - /// The location of the error. - /// An optional stack trace of the error, or null. - /// The severity of the error. - public void ExtractionError(string message, string? entityText, CSharp.Entities.Location? location, string? stackTrace = null, Severity severity = Severity.Error) - { - var msg = new Message(message, entityText, location, stackTrace, severity); - ExtractionError(msg); - } - - /// - /// Log an extraction error. - /// - /// The text of the message. - /// The symbol of the error, or null. - /// The entity of the error, or null. - private void ExtractionError(string message, ISymbol? optionalSymbol, Entity optionalEntity) - { - if (!(optionalSymbol is null)) - { - ExtractionError(message, optionalSymbol.ToDisplayString(), CreateLocation(optionalSymbol.Locations.BestOrDefault())); - } - else if (!(optionalEntity is null)) - { - ExtractionError(message, optionalEntity.Label.ToString(), CreateLocation(optionalEntity.ReportingLocation)); - } - else - { - ExtractionError(message, null, CreateLocation()); - } - } - - /// - /// Log an extraction message. - /// - /// The message to log. - private void ExtractionError(Message msg) - { - new ExtractionMessage(this, msg); - ExtractionContext.Message(msg); - } - - private void ExtractionError(InternalError error) - { - ExtractionError(new Message(error.Message, error.EntityText, CreateLocation(error.Location), error.StackTrace, Severity.Error)); - } - - private void ReportError(InternalError error) - { - if (!ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone)) - throw error; - - ExtractionError(error); - } - - /// - /// Signal an error in the program model. - /// - /// The syntax node causing the failure. - /// The error message. - public void ModelError(SyntaxNode node, string msg) - { - ReportError(new InternalError(node, msg)); - } - - /// - /// Signal an error in the program model. - /// - /// Symbol causing the error. - /// The error message. - public void ModelError(ISymbol symbol, string msg) - { - ReportError(new InternalError(symbol, msg)); - } - - /// - /// Signal an error in the program model. - /// - /// The location of the error. - /// The error message. - public void ModelError(CSharp.Entities.Location loc, string msg) - { - ReportError(new InternalError(loc.ReportingLocation, msg)); - } - - /// - /// Signal an error in the program model. - /// - /// The error message. - public void ModelError(string msg) - { - ReportError(new InternalError(msg)); - } - - /// - /// Tries the supplied action , and logs an uncaught - /// exception error if the action fails. - /// - /// Optional syntax node for error reporting. - /// Optional symbol for error reporting. - /// The action to perform. - public void Try(SyntaxNode? node, ISymbol? symbol, Action a) - { - try - { - a(); - } - catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] - { - Message message; - - if (node is not null) - { - message = Message.Create(this, ex.Message, node, ex.StackTrace); - } - else if (symbol is not null) - { - message = Message.Create(this, ex.Message, symbol, ex.StackTrace); - } - else if (ex is InternalError ie) - { - message = new Message(ie.Text, ie.EntityText, CreateLocation(ie.Location), ex.StackTrace); - } - else - { - message = new Message($"Uncaught exception. {ex.Message}", null, CreateLocation(), ex.StackTrace); - } - - ExtractionError(message); - } - } - - public virtual CSharp.Entities.Location CreateLocation() => - CSharp.Entities.GeneratedLocation.Create((CSharp.Context)this); - - public virtual CSharp.Entities.Location CreateLocation(Microsoft.CodeAnalysis.Location? location) => - CreateLocation(); - } -}