Added support for Stream Markers.

Src/ILGPU/Resources/RuntimeErrorMessages.resx

@@ -241,9 +241,12 @@
     <value>Unknown parent accelerator</value>
   </data>
   <data name="VelocityPlatform64" xml:space="preserve">
-      <value>Velocity accelerator requires 64-bit application ({0} not supported). Ensure Prefer32Bit is set to 'false'</value>
+    <value>Velocity accelerator requires 64-bit application ({0} not supported). Ensure Prefer32Bit is set to 'false'</value>
   </data>
   <data name="VelocityLittleEndian" xml:space="preserve">
-      <value>The Velocity accelerator supports little-endian machines only</value>
+    <value>The Velocity accelerator supports little-endian machines only</value>
+  </data>
+  <data name="NotSupportedAcceleratorStreamMarker" xml:space="preserve">
+    <value>The accelerator stream marker is not supported for this operation</value>
   </data>
 </root>

Src/ILGPU/Runtime/Accelerator.cs

@@ -1,6 +1,6 @@
 // ---------------------------------------------------------------------------------------
 //                                        ILGPU
-//                        Copyright (c) 2016-2023 ILGPU Project
+//                        Copyright (c) 2016-2024 ILGPU Project
 //                                    www.ilgpu.net
 //
 // File: Accelerator.cs
@@ -16,6 +16,7 @@
 using System;
 using System.Diagnostics;
 using System.IO;
+using System.Runtime.CompilerServices;
 using System.Threading;
 
 namespace ILGPU.Runtime
@@ -263,6 +264,22 @@ public override void ClearCache(ClearCacheMode mode)
                 base.ClearCache(mode);
             }
         }
+        /// <summary>
+        /// Creates a new accelerator stream marker.
+        /// </summary>
+        /// <returns>The created accelerator stream marker.</returns>
+        public StreamMarker CreateStreamMarker()
+        {
+            Bind();
+            return CreateStreamMarkerInternal();
+        }
+
+        /// <summary>
+        /// Creates a new accelerator stream marker.
+        /// </summary>
+        /// <returns>The created accelerator stream marker.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        protected abstract StreamMarker CreateStreamMarkerInternal();
 
         #endregion
 

Src/ILGPU/Runtime/AcceleratorStream.cs

@@ -1,6 +1,6 @@
 // ---------------------------------------------------------------------------------------
 //                                        ILGPU
-//                        Copyright (c) 2017-2023 ILGPU Project
+//                        Copyright (c) 2017-2024 ILGPU Project
 //                                    www.ilgpu.net
 //
 // File: AcceleratorStream.cs
@@ -80,6 +80,23 @@ public ProfilingMarker AddProfilingMarker() =>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         protected abstract ProfilingMarker AddProfilingMarkerInternal();
 
+        /// <summary>
+        /// Makes future work queued on this stream to wait for the marker to complete.
+        /// </summary>
+        /// <param name="streamMarker">The stream marker to await.</param>
+        public void WaitForStreamMarker(StreamMarker streamMarker)
+        {
+            using var binding = BindScoped();
+            WaitForStreamMarkerInternal(streamMarker);
+        }
+
+        /// <summary>
+        /// Makes future work queued on this stream to wait for the marker to complete.
+        /// </summary>
+        /// <param name="streamMarker">The stream marker to await.</param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        protected abstract void WaitForStreamMarkerInternal(StreamMarker streamMarker);
+
         #endregion
     }
 }

Src/ILGPU/Runtime/CPU/CPUAccelerator.cs

@@ -1,6 +1,6 @@
 // ---------------------------------------------------------------------------------------
 //                                        ILGPU
-//                        Copyright (c) 2016-2023 ILGPU Project
+//                        Copyright (c) 2016-2024 ILGPU Project
 //                                    www.ilgpu.net
 //
 // File: CPUAccelerator.cs
@@ -246,6 +246,10 @@ protected override void OnBind() { }
         /// <summary cref="Accelerator.OnUnbind"/>
         protected override void OnUnbind() { }
 
+        /// <inheritdoc/>
+        protected override StreamMarker CreateStreamMarkerInternal() =>
+            new CPUStreamMarker(this);
+
         #endregion
 
         #region Peer Access

Src/ILGPU/Runtime/CPU/CPUStream.cs

@@ -1,6 +1,6 @@
 // ---------------------------------------------------------------------------------------
 //                                        ILGPU
-//                        Copyright (c) 2017-2023 ILGPU Project
+//                        Copyright (c) 2017-2024 ILGPU Project
 //                                    www.ilgpu.net
 //
 // File: CPUStream.cs
@@ -9,6 +9,9 @@
 // Source License. See LICENSE.txt for details.
 // ---------------------------------------------------------------------------------------
 
+using ILGPU.Resources;
+using System;
+
 namespace ILGPU.Runtime.CPU
 {
     /// <summary>
@@ -42,6 +45,17 @@ protected unsafe override ProfilingMarker AddProfilingMarkerInternal()
             return new CPUProfilingMarker(Accelerator);
         }
 
+        /// <inheritdoc/>
+        protected unsafe override void WaitForStreamMarkerInternal(
+            StreamMarker streamMarker)
+        {
+            if (streamMarker is not CPUStreamMarker)
+            {
+                throw new NotSupportedException(
+                    RuntimeErrorMessages.NotSupportedAcceleratorStreamMarker);
+            }
+        }
+
         #endregion
 
         #region IDisposable

Src/ILGPU/Runtime/CPU/CPUStreamMarker.cs

@@ -0,0 +1,50 @@
+// ---------------------------------------------------------------------------------------
+//                                        ILGPU
+//                           Copyright (c) 2024 ILGPU Project
+//                                    www.ilgpu.net
+//
+// File: CPUStreamMarker.cs
+//
+// This file is part of ILGPU and is distributed under the University of Illinois Open
+// Source License. See LICENSE.txt for details.
+// ---------------------------------------------------------------------------------------
+
+using ILGPU.Resources;
+using System;
+
+namespace ILGPU.Runtime.CPU
+{
+    /// <summary>
+    /// Represents a marker used in CPU streams.
+    /// </summary>
+    internal sealed class CPUStreamMarker : StreamMarker
+    {
+        #region Instance
+
+        internal CPUStreamMarker(Accelerator accelerator)
+            : base(accelerator)
+        { }
+
+        #endregion
+
+        #region Methods
+
+        /// <inheritdoc/>
+        public override void Synchronize() { }
+
+        /// <inheritdoc/>
+        protected override void DisposeAcceleratorObject(bool disposing) { }
+
+        /// <inheritdoc/>
+        public unsafe override void Record(AcceleratorStream stream)
+        {
+            if (stream is not CPUStream)
+            {
+                throw new NotSupportedException(
+                    RuntimeErrorMessages.NotSupportedAcceleratorStream);
+            }
+        }
+
+        #endregion
+    }
+}

Src/ILGPU/Runtime/Cuda/CudaAPI.cs

@@ -1,6 +1,6 @@
 // ---------------------------------------------------------------------------------------
 //                                        ILGPU
-//                        Copyright (c) 2020-2023 ILGPU Project
+//                        Copyright (c) 2020-2024 ILGPU Project
 //                                    www.ilgpu.net
 //
 // File: CudaAPI.cs
@@ -528,6 +528,16 @@ public CudaError DestroyStream(IntPtr stream) =>
         public CudaError SynchronizeStream(IntPtr stream) =>
             cuStreamSynchronize(stream);
 
+        /// <summary>
+        /// Make the stream wait for the event.
+        /// </summary>
+        /// <param name="stream">The stream to wait.</param>
+        /// <param name="event">The event to await.</param>
+        /// <param name="flags">The flags to use.</param>
+        /// <returns>The error status.</returns>
+        public CudaError WaitForEvent(IntPtr stream, IntPtr @event, IntPtr flags) =>
+            cuStreamWaitEvent(stream, @event, flags);
+
         #endregion
 
         #region Kernel Methods

Src/ILGPU/Runtime/Cuda/CudaAPI.xml

@@ -141,6 +141,11 @@
     <Import Name="cuStreamSynchronize">
         <Parameter Name="stream" Type="IntPtr" />
     </Import>
+    <Import Name="cuStreamWaitEvent">
+        <Parameter Name="stream" Type="IntPtr" />
+        <Parameter Name="@event" Type="IntPtr" />
+        <Parameter Name="flags" Type="IntPtr" />
+    </Import>
     <Import Name="cuGetErrorString">
         <Parameter Name="error" Type="CudaError" />
         <Parameter Name="pStr" Type="IntPtr" Flags="Out" />

Src/ILGPU/Runtime/Cuda/CudaAccelerator.cs

@@ -1,6 +1,6 @@
 // ---------------------------------------------------------------------------------------
 //                                        ILGPU
-//                        Copyright (c) 2016-2023 ILGPU Project
+//                        Copyright (c) 2016-2024 ILGPU Project
 //                                    www.ilgpu.net
 //
 // File: CudaAccelerator.cs
@@ -440,6 +440,10 @@ public long GetFreeMemory()
             return free;
         }
 
+        /// <inheritdoc/>
+        protected override StreamMarker CreateStreamMarkerInternal() =>
+            new CudaStreamMarker(this);
+
         #endregion
 
         #region Allocation

Src/ILGPU/Runtime/Cuda/CudaStream.cs

@@ -1,6 +1,6 @@
 // ---------------------------------------------------------------------------------------
 //                                        ILGPU
-//                        Copyright (c) 2017-2021 ILGPU Project
+//                        Copyright (c) 2017-2024 ILGPU Project
 //                                    www.ilgpu.net
 //
 // File: CudaStream.cs
@@ -9,6 +9,7 @@
 // Source License. See LICENSE.txt for details.
 // ---------------------------------------------------------------------------------------
 
+using ILGPU.Resources;
 using System;
 using System.Diagnostics.CodeAnalysis;
 using static ILGPU.Runtime.Cuda.CudaAPI;
@@ -95,6 +96,24 @@ protected override ProfilingMarker AddProfilingMarkerInternal()
             return profilingMarker;
         }
 
+        /// <inheritdoc/>
+        protected unsafe override void WaitForStreamMarkerInternal(
+            StreamMarker streamMarker)
+        {
+            if (streamMarker is not CudaStreamMarker cudaStreamMarker)
+            {
+                throw new NotSupportedException(
+                    RuntimeErrorMessages.NotSupportedAcceleratorStreamMarker);
+            }
+
+            using var binding = BindScoped();
+            CudaException.ThrowIfFailed(
+                CurrentAPI.WaitForEvent(
+                    StreamPtr,
+                    cudaStreamMarker.EventPtr,
+                    IntPtr.Zero));
+        }
+
         #endregion
 
         #region IDisposable

Src/ILGPU/Runtime/Cuda/CudaStreamMarker.cs

@@ -0,0 +1,84 @@
+// ---------------------------------------------------------------------------------------
+//                                        ILGPU
+//                           Copyright (c) 2024 ILGPU Project
+//                                    www.ilgpu.net
+//
+// File: CudaStreamMarker.cs
+//
+// This file is part of ILGPU and is distributed under the University of Illinois Open
+// Source License. See LICENSE.txt for details.
+// ---------------------------------------------------------------------------------------
+
+using ILGPU.Resources;
+using System;
+using static ILGPU.Runtime.Cuda.CudaAPI;
+
+namespace ILGPU.Runtime.Cuda
+{
+    /// <summary>
+    /// Represents a marker used in CUDA streams.
+    /// </summary>
+    internal sealed class CudaStreamMarker : StreamMarker
+    {
+        #region Instance
+
+        internal CudaStreamMarker(Accelerator accelerator)
+            : base (accelerator)
+        {
+            CudaException.ThrowIfFailed(
+                CurrentAPI.CreateEvent(
+                    out var eventPtr,
+                    CudaEventFlags.CU_EVENT_DEFAULT));
+            EventPtr = eventPtr;
+        }
+
+        #endregion
+
+        #region Properties
+
+        /// <summary>
+        /// The native event pointer.
+        /// </summary>
+        public IntPtr EventPtr { get; private set; }
+
+        #endregion
+
+        #region Methods
+
+        /// <inheritdoc/>
+        public override void Synchronize()
+        {
+            using var binding = Accelerator.BindScoped();
+
+            var errorStatus = CurrentAPI.QueryEvent(EventPtr);
+            if (errorStatus == CudaError.CUDA_ERROR_NOT_READY)
+                CudaException.ThrowIfFailed(CurrentAPI.SynchronizeEvent(EventPtr));
+            else
+                CudaException.ThrowIfFailed(errorStatus);
+        }
+
+        /// <inheritdoc/>
+        protected override void DisposeAcceleratorObject(bool disposing)
+        {
+            CudaException.VerifyDisposed(
+                disposing,
+                CurrentAPI.DestroyEvent(EventPtr));
+            EventPtr = IntPtr.Zero;
+        }
+
+        /// <inheritdoc/>
+        public unsafe override void Record(AcceleratorStream stream)
+        {
+            if (stream is not CudaStream cudaStream)
+            {
+                throw new NotSupportedException(
+                    RuntimeErrorMessages.NotSupportedAcceleratorStream);
+            }
+
+            CudaException.ThrowIfFailed(
+                CurrentAPI.RecordEvent(EventPtr, cudaStream.StreamPtr));
+        }
+
+        #endregion
+    }
+}