diff --git a/Src/ILGPU/Resources/RuntimeErrorMessages.Designer.cs b/Src/ILGPU/Resources/RuntimeErrorMessages.Designer.cs index 7c6931ffb..1d99f8f7b 100644 --- a/Src/ILGPU/Resources/RuntimeErrorMessages.Designer.cs +++ b/Src/ILGPU/Resources/RuntimeErrorMessages.Designer.cs @@ -246,6 +246,15 @@ internal static string NotSupportedAcceleratorStream { } } + /// + /// Looks up a localized string similar to The accelerator stream marker is not supported for this operation. + /// + internal static string NotSupportedAcceleratorStreamMarker { + get { + return ResourceManager.GetString("NotSupportedAcceleratorStreamMarker", resourceCulture); + } + } + /// /// Looks up a localized string similar to Not supported driver version '{0}' (>= {1} required). /// diff --git a/Src/ILGPU/Resources/RuntimeErrorMessages.resx b/Src/ILGPU/Resources/RuntimeErrorMessages.resx index d2b8d8819..6daeb3ace 100644 --- a/Src/ILGPU/Resources/RuntimeErrorMessages.resx +++ b/Src/ILGPU/Resources/RuntimeErrorMessages.resx @@ -241,9 +241,12 @@ Unknown parent accelerator - Velocity accelerator requires 64-bit application ({0} not supported). Ensure Prefer32Bit is set to 'false' + Velocity accelerator requires 64-bit application ({0} not supported). Ensure Prefer32Bit is set to 'false' - The Velocity accelerator supports little-endian machines only + The Velocity accelerator supports little-endian machines only + + + The accelerator stream marker is not supported for this operation \ No newline at end of file diff --git a/Src/ILGPU/Runtime/Accelerator.cs b/Src/ILGPU/Runtime/Accelerator.cs index 0334c743f..dd8f64358 100644 --- a/Src/ILGPU/Runtime/Accelerator.cs +++ b/Src/ILGPU/Runtime/Accelerator.cs @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------------------- // ILGPU -// Copyright (c) 2016-2023 ILGPU Project +// Copyright (c) 2016-2024 ILGPU Project // www.ilgpu.net // // File: Accelerator.cs @@ -16,6 +16,7 @@ using System; using System.Diagnostics; using System.IO; +using System.Runtime.CompilerServices; using System.Threading; namespace ILGPU.Runtime @@ -263,6 +264,22 @@ public override void ClearCache(ClearCacheMode mode) base.ClearCache(mode); } } + /// + /// Creates a new accelerator stream marker. + /// + /// The created accelerator stream marker. + public StreamMarker CreateStreamMarker() + { + Bind(); + return CreateStreamMarkerInternal(); + } + + /// + /// Creates a new accelerator stream marker. + /// + /// The created accelerator stream marker. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected abstract StreamMarker CreateStreamMarkerInternal(); #endregion diff --git a/Src/ILGPU/Runtime/AcceleratorStream.cs b/Src/ILGPU/Runtime/AcceleratorStream.cs index 542c7655b..4fd42dccd 100644 --- a/Src/ILGPU/Runtime/AcceleratorStream.cs +++ b/Src/ILGPU/Runtime/AcceleratorStream.cs @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------------------- // ILGPU -// Copyright (c) 2017-2023 ILGPU Project +// Copyright (c) 2017-2024 ILGPU Project // www.ilgpu.net // // File: AcceleratorStream.cs @@ -80,6 +80,23 @@ public ProfilingMarker AddProfilingMarker() => [MethodImpl(MethodImplOptions.AggressiveInlining)] protected abstract ProfilingMarker AddProfilingMarkerInternal(); + /// + /// Makes future work queued on this stream to wait for the marker to complete. + /// + /// The stream marker to await. + public void WaitForStreamMarker(StreamMarker streamMarker) + { + using var binding = BindScoped(); + WaitForStreamMarkerInternal(streamMarker); + } + + /// + /// Makes future work queued on this stream to wait for the marker to complete. + /// + /// The stream marker to await. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected abstract void WaitForStreamMarkerInternal(StreamMarker streamMarker); + #endregion } } diff --git a/Src/ILGPU/Runtime/CPU/CPUAccelerator.cs b/Src/ILGPU/Runtime/CPU/CPUAccelerator.cs index d39252caa..9a94dedef 100644 --- a/Src/ILGPU/Runtime/CPU/CPUAccelerator.cs +++ b/Src/ILGPU/Runtime/CPU/CPUAccelerator.cs @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------------------- // ILGPU -// Copyright (c) 2016-2023 ILGPU Project +// Copyright (c) 2016-2024 ILGPU Project // www.ilgpu.net // // File: CPUAccelerator.cs @@ -246,6 +246,10 @@ protected override void OnBind() { } /// protected override void OnUnbind() { } + /// + protected override StreamMarker CreateStreamMarkerInternal() => + new CPUStreamMarker(this); + #endregion #region Peer Access diff --git a/Src/ILGPU/Runtime/CPU/CPUStream.cs b/Src/ILGPU/Runtime/CPU/CPUStream.cs index 4778cc882..915bbc1c3 100644 --- a/Src/ILGPU/Runtime/CPU/CPUStream.cs +++ b/Src/ILGPU/Runtime/CPU/CPUStream.cs @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------------------- // ILGPU -// Copyright (c) 2017-2023 ILGPU Project +// Copyright (c) 2017-2024 ILGPU Project // www.ilgpu.net // // File: CPUStream.cs @@ -9,6 +9,9 @@ // Source License. See LICENSE.txt for details. // --------------------------------------------------------------------------------------- +using ILGPU.Resources; +using System; + namespace ILGPU.Runtime.CPU { /// @@ -42,6 +45,17 @@ protected unsafe override ProfilingMarker AddProfilingMarkerInternal() return new CPUProfilingMarker(Accelerator); } + /// + protected unsafe override void WaitForStreamMarkerInternal( + StreamMarker streamMarker) + { + if (streamMarker is not CPUStreamMarker) + { + throw new NotSupportedException( + RuntimeErrorMessages.NotSupportedAcceleratorStreamMarker); + } + } + #endregion #region IDisposable diff --git a/Src/ILGPU/Runtime/CPU/CPUStreamMarker.cs b/Src/ILGPU/Runtime/CPU/CPUStreamMarker.cs new file mode 100644 index 000000000..890bb3b0f --- /dev/null +++ b/Src/ILGPU/Runtime/CPU/CPUStreamMarker.cs @@ -0,0 +1,50 @@ +// --------------------------------------------------------------------------------------- +// ILGPU +// Copyright (c) 2024 ILGPU Project +// www.ilgpu.net +// +// File: CPUStreamMarker.cs +// +// This file is part of ILGPU and is distributed under the University of Illinois Open +// Source License. See LICENSE.txt for details. +// --------------------------------------------------------------------------------------- + +using ILGPU.Resources; +using System; + +namespace ILGPU.Runtime.CPU +{ + /// + /// Represents a marker used in CPU streams. + /// + internal sealed class CPUStreamMarker : StreamMarker + { + #region Instance + + internal CPUStreamMarker(Accelerator accelerator) + : base(accelerator) + { } + + #endregion + + #region Methods + + /// + public override void Synchronize() { } + + /// + protected override void DisposeAcceleratorObject(bool disposing) { } + + /// + public unsafe override void Record(AcceleratorStream stream) + { + if (stream is not CPUStream) + { + throw new NotSupportedException( + RuntimeErrorMessages.NotSupportedAcceleratorStream); + } + } + + #endregion + } +} diff --git a/Src/ILGPU/Runtime/Cuda/CudaAPI.cs b/Src/ILGPU/Runtime/Cuda/CudaAPI.cs index 98e6053d7..be2631f65 100644 --- a/Src/ILGPU/Runtime/Cuda/CudaAPI.cs +++ b/Src/ILGPU/Runtime/Cuda/CudaAPI.cs @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------------------- // ILGPU -// Copyright (c) 2020-2023 ILGPU Project +// Copyright (c) 2020-2024 ILGPU Project // www.ilgpu.net // // File: CudaAPI.cs @@ -528,6 +528,16 @@ public CudaError DestroyStream(IntPtr stream) => public CudaError SynchronizeStream(IntPtr stream) => cuStreamSynchronize(stream); + /// + /// Make the stream wait for the event. + /// + /// The stream to wait. + /// The event to await. + /// The flags to use. + /// The error status. + public CudaError WaitForEvent(IntPtr stream, IntPtr @event, IntPtr flags) => + cuStreamWaitEvent(stream, @event, flags); + #endregion #region Kernel Methods diff --git a/Src/ILGPU/Runtime/Cuda/CudaAPI.xml b/Src/ILGPU/Runtime/Cuda/CudaAPI.xml index ea0913437..9ef3f6dfa 100644 --- a/Src/ILGPU/Runtime/Cuda/CudaAPI.xml +++ b/Src/ILGPU/Runtime/Cuda/CudaAPI.xml @@ -141,6 +141,11 @@ + + + + + diff --git a/Src/ILGPU/Runtime/Cuda/CudaAccelerator.cs b/Src/ILGPU/Runtime/Cuda/CudaAccelerator.cs index c338c10fa..5dc43216d 100644 --- a/Src/ILGPU/Runtime/Cuda/CudaAccelerator.cs +++ b/Src/ILGPU/Runtime/Cuda/CudaAccelerator.cs @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------------------- // ILGPU -// Copyright (c) 2016-2023 ILGPU Project +// Copyright (c) 2016-2024 ILGPU Project // www.ilgpu.net // // File: CudaAccelerator.cs @@ -440,6 +440,10 @@ public long GetFreeMemory() return free; } + /// + protected override StreamMarker CreateStreamMarkerInternal() => + new CudaStreamMarker(this); + #endregion #region Allocation diff --git a/Src/ILGPU/Runtime/Cuda/CudaStream.cs b/Src/ILGPU/Runtime/Cuda/CudaStream.cs index a2d307fb9..8244f309e 100644 --- a/Src/ILGPU/Runtime/Cuda/CudaStream.cs +++ b/Src/ILGPU/Runtime/Cuda/CudaStream.cs @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------------------- // ILGPU -// Copyright (c) 2017-2021 ILGPU Project +// Copyright (c) 2017-2024 ILGPU Project // www.ilgpu.net // // File: CudaStream.cs @@ -9,6 +9,7 @@ // Source License. See LICENSE.txt for details. // --------------------------------------------------------------------------------------- +using ILGPU.Resources; using System; using System.Diagnostics.CodeAnalysis; using static ILGPU.Runtime.Cuda.CudaAPI; @@ -95,6 +96,24 @@ protected override ProfilingMarker AddProfilingMarkerInternal() return profilingMarker; } + /// + protected unsafe override void WaitForStreamMarkerInternal( + StreamMarker streamMarker) + { + if (streamMarker is not CudaStreamMarker cudaStreamMarker) + { + throw new NotSupportedException( + RuntimeErrorMessages.NotSupportedAcceleratorStreamMarker); + } + + using var binding = BindScoped(); + CudaException.ThrowIfFailed( + CurrentAPI.WaitForEvent( + StreamPtr, + cudaStreamMarker.EventPtr, + IntPtr.Zero)); + } + #endregion #region IDisposable diff --git a/Src/ILGPU/Runtime/Cuda/CudaStreamMarker.cs b/Src/ILGPU/Runtime/Cuda/CudaStreamMarker.cs new file mode 100644 index 000000000..be1b283e5 --- /dev/null +++ b/Src/ILGPU/Runtime/Cuda/CudaStreamMarker.cs @@ -0,0 +1,84 @@ +// --------------------------------------------------------------------------------------- +// ILGPU +// Copyright (c) 2024 ILGPU Project +// www.ilgpu.net +// +// File: CudaStreamMarker.cs +// +// This file is part of ILGPU and is distributed under the University of Illinois Open +// Source License. See LICENSE.txt for details. +// --------------------------------------------------------------------------------------- + +using ILGPU.Resources; +using System; +using static ILGPU.Runtime.Cuda.CudaAPI; + +namespace ILGPU.Runtime.Cuda +{ + /// + /// Represents a marker used in CUDA streams. + /// + internal sealed class CudaStreamMarker : StreamMarker + { + #region Instance + + internal CudaStreamMarker(Accelerator accelerator) + : base (accelerator) + { + CudaException.ThrowIfFailed( + CurrentAPI.CreateEvent( + out var eventPtr, + CudaEventFlags.CU_EVENT_DEFAULT)); + EventPtr = eventPtr; + } + + #endregion + + #region Properties + + /// + /// The native event pointer. + /// + public IntPtr EventPtr { get; private set; } + + #endregion + + #region Methods + + /// + public override void Synchronize() + { + using var binding = Accelerator.BindScoped(); + + var errorStatus = CurrentAPI.QueryEvent(EventPtr); + if (errorStatus == CudaError.CUDA_ERROR_NOT_READY) + CudaException.ThrowIfFailed(CurrentAPI.SynchronizeEvent(EventPtr)); + else + CudaException.ThrowIfFailed(errorStatus); + } + + /// + protected override void DisposeAcceleratorObject(bool disposing) + { + CudaException.VerifyDisposed( + disposing, + CurrentAPI.DestroyEvent(EventPtr)); + EventPtr = IntPtr.Zero; + } + + /// + public unsafe override void Record(AcceleratorStream stream) + { + if (stream is not CudaStream cudaStream) + { + throw new NotSupportedException( + RuntimeErrorMessages.NotSupportedAcceleratorStream); + } + + CudaException.ThrowIfFailed( + CurrentAPI.RecordEvent(EventPtr, cudaStream.StreamPtr)); + } + + #endregion + } +} diff --git a/Src/ILGPU/Runtime/OpenCL/CLAccelerator.cs b/Src/ILGPU/Runtime/OpenCL/CLAccelerator.cs index 8a83b7256..f5592a1c6 100644 --- a/Src/ILGPU/Runtime/OpenCL/CLAccelerator.cs +++ b/Src/ILGPU/Runtime/OpenCL/CLAccelerator.cs @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------------------- // ILGPU -// Copyright (c) 2019-2023 ILGPU Project +// Copyright (c) 2019-2024 ILGPU Project // www.ilgpu.net // // File: CLAccelerator.cs @@ -393,6 +393,10 @@ protected override void OnBind() { } /// protected override void OnUnbind() { } + /// + protected override StreamMarker CreateStreamMarkerInternal() => + new CLStreamMarker(this); + #endregion #region Peer Access diff --git a/Src/ILGPU/Runtime/OpenCL/CLStream.cs b/Src/ILGPU/Runtime/OpenCL/CLStream.cs index 648d79561..c1ce7b56a 100644 --- a/Src/ILGPU/Runtime/OpenCL/CLStream.cs +++ b/Src/ILGPU/Runtime/OpenCL/CLStream.cs @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------------------- // ILGPU -// Copyright (c) 2019-2021 ILGPU Project +// Copyright (c) 2019-2024 ILGPU Project // www.ilgpu.net // // File: CLStream.cs @@ -9,6 +9,7 @@ // Source License. See LICENSE.txt for details. // --------------------------------------------------------------------------------------- +using ILGPU.Resources; using System; using System.Diagnostics.CodeAnalysis; using static ILGPU.Runtime.OpenCL.CLAPI; @@ -93,6 +94,26 @@ protected unsafe override ProfilingMarker AddProfilingMarkerInternal() return marker; } + /// + protected unsafe override void WaitForStreamMarkerInternal( + StreamMarker streamMarker) + { + if (streamMarker is not CLStreamMarker clStreamMarker || + clStreamMarker.EventPtr == IntPtr.Zero) + { + throw new NotSupportedException( + RuntimeErrorMessages.NotSupportedAcceleratorStreamMarker); + } + + using var binding = BindScoped(); + var streamEvents = new IntPtr[] { clStreamMarker.EventPtr }; + CLException.ThrowIfFailed( + CurrentAPI.EnqueueBarrierWithWaitList( + CommandQueue, + streamEvents, + null)); + } + #endregion #region IDisposable diff --git a/Src/ILGPU/Runtime/OpenCL/CLStreamMarker.cs b/Src/ILGPU/Runtime/OpenCL/CLStreamMarker.cs new file mode 100644 index 000000000..b35642753 --- /dev/null +++ b/Src/ILGPU/Runtime/OpenCL/CLStreamMarker.cs @@ -0,0 +1,99 @@ +// --------------------------------------------------------------------------------------- +// ILGPU +// Copyright (c) 2024 ILGPU Project +// www.ilgpu.net +// +// File: CLStreamMarker.cs +// +// This file is part of ILGPU and is distributed under the University of Illinois Open +// Source License. See LICENSE.txt for details. +// --------------------------------------------------------------------------------------- + +using ILGPU.Resources; +using System; +using static ILGPU.Runtime.OpenCL.CLAPI; + +namespace ILGPU.Runtime.OpenCL +{ + /// + /// Represents a marker used in OpenCL streams. + /// + internal sealed class CLStreamMarker : StreamMarker + { + #region Instance + + internal CLStreamMarker(Accelerator accelerator) + : base(accelerator) + { + EventPtr = IntPtr.Zero; + } + + #endregion + + #region Properties + + /// + /// The native event pointer. + /// + public IntPtr EventPtr { get; private set; } + + #endregion + + #region Methods + + /// + public unsafe override void Synchronize() + { + if (EventPtr == IntPtr.Zero) + { + throw new NotSupportedException( + RuntimeErrorMessages.NotSupportedAcceleratorStreamMarker); + } + + using var binding = Accelerator.BindScoped(); + + ReadOnlySpan events = stackalloc[] { EventPtr }; + CLException.ThrowIfFailed( + CurrentAPI.WaitForEvents(events)); + } + + /// + protected override void DisposeAcceleratorObject(bool disposing) + { + CLException.VerifyDisposed( + disposing, + CurrentAPI.clReleaseEvent(EventPtr)); + EventPtr = IntPtr.Zero; + } + + /// + public unsafe override void Record(AcceleratorStream stream) + { + if (stream is not CLStream clStream) + { + throw new NotSupportedException( + RuntimeErrorMessages.NotSupportedAcceleratorStream); + } + + // If we have a previously recorded event, discard it. + if (EventPtr != IntPtr.Zero) + { + CLException.ThrowIfFailed(CurrentAPI.clReleaseEvent(EventPtr)); + EventPtr = IntPtr.Zero; + } + + // Waits for all previously enqueued commands to finish, and then completes + // the event. We can therefore use the event as a marker to record the current + // state of the queue. + IntPtr* streamEvent = stackalloc IntPtr[1]; + CLException.ThrowIfFailed( + CurrentAPI.EnqueueBarrierWithWaitList( + clStream.CommandQueue, + Array.Empty(), + streamEvent)); + EventPtr = streamEvent[0]; + } + + #endregion + } +} diff --git a/Src/ILGPU/Runtime/StreamMarker.cs b/Src/ILGPU/Runtime/StreamMarker.cs new file mode 100644 index 000000000..d415abcd4 --- /dev/null +++ b/Src/ILGPU/Runtime/StreamMarker.cs @@ -0,0 +1,37 @@ +// --------------------------------------------------------------------------------------- +// ILGPU +// Copyright (c) 2024 ILGPU Project +// www.ilgpu.net +// +// File: StreamMarker.cs +// +// This file is part of ILGPU and is distributed under the University of Illinois Open +// Source License. See LICENSE.txt for details. +// --------------------------------------------------------------------------------------- + +namespace ILGPU.Runtime +{ + /// + /// Represents a marker used in a stream. + /// + public abstract class StreamMarker : AcceleratorObject + { + /// + /// Constructs a stream marker. + /// + /// The associated accelerator. + protected StreamMarker(Accelerator accelerator) + : base(accelerator) + { } + + /// + /// Captures the contents of the stream. + /// + public abstract void Record(AcceleratorStream stream); + + /// + /// Waits for the stream marker to complete. + /// + public abstract void Synchronize(); + } +} diff --git a/Src/ILGPU/Runtime/Velocity/VelocityAccelerator.cs b/Src/ILGPU/Runtime/Velocity/VelocityAccelerator.cs index 7ba2dbb26..83e8681e5 100644 --- a/Src/ILGPU/Runtime/Velocity/VelocityAccelerator.cs +++ b/Src/ILGPU/Runtime/Velocity/VelocityAccelerator.cs @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------------------- // ILGPU -// Copyright (c) 2022-2023 ILGPU Project +// Copyright (c) 2022-2024 ILGPU Project // www.ilgpu.net // // File: VelocityAccelerator.cs @@ -464,6 +464,10 @@ protected override void OnBind() { } /// protected override void OnUnbind() { } + /// + protected override StreamMarker CreateStreamMarkerInternal() => + new VelocityStreamMarker(this); + #region Peer Access /// diff --git a/Src/ILGPU/Runtime/Velocity/VelocityStream.cs b/Src/ILGPU/Runtime/Velocity/VelocityStream.cs index 4ea432842..288f25708 100644 --- a/Src/ILGPU/Runtime/Velocity/VelocityStream.cs +++ b/Src/ILGPU/Runtime/Velocity/VelocityStream.cs @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------------------- // ILGPU -// Copyright (c) 2022-2023 ILGPU Project +// Copyright (c) 2022-2024 ILGPU Project // www.ilgpu.net // // File: VelocityStream.cs @@ -9,6 +9,9 @@ // Source License. See LICENSE.txt for details. // --------------------------------------------------------------------------------------- +using ILGPU.Resources; +using System; + namespace ILGPU.Runtime.Velocity { /// @@ -42,6 +45,17 @@ protected override ProfilingMarker AddProfilingMarkerInternal() return new VelocityProfilingMarker(Accelerator); } + /// + protected unsafe override void WaitForStreamMarkerInternal( + StreamMarker streamMarker) + { + if (streamMarker is not VelocityStreamMarker) + { + throw new NotSupportedException( + RuntimeErrorMessages.NotSupportedAcceleratorStreamMarker); + } + } + #endregion #region IDisposable diff --git a/Src/ILGPU/Runtime/Velocity/VelocityStreamMarker.cs b/Src/ILGPU/Runtime/Velocity/VelocityStreamMarker.cs new file mode 100644 index 000000000..463d39740 --- /dev/null +++ b/Src/ILGPU/Runtime/Velocity/VelocityStreamMarker.cs @@ -0,0 +1,50 @@ +// --------------------------------------------------------------------------------------- +// ILGPU +// Copyright (c) 2024 ILGPU Project +// www.ilgpu.net +// +// File: VelocityStreamMarker.cs +// +// This file is part of ILGPU and is distributed under the University of Illinois Open +// Source License. See LICENSE.txt for details. +// --------------------------------------------------------------------------------------- + +using ILGPU.Resources; +using System; + +namespace ILGPU.Runtime.Velocity +{ + /// + /// Represents a marker used in velocity streams. + /// + internal sealed class VelocityStreamMarker : StreamMarker + { + #region Instance + + internal VelocityStreamMarker(Accelerator accelerator) + : base(accelerator) + { } + + #endregion + + #region Methods + + /// + public override void Synchronize() { } + + /// + protected override void DisposeAcceleratorObject(bool disposing) { } + + /// + public unsafe override void Record(AcceleratorStream stream) + { + if (stream is not VelocityStream) + { + throw new NotSupportedException( + RuntimeErrorMessages.NotSupportedAcceleratorStream); + } + } + + #endregion + } +}