From 039d681cc7e12cf2fdd8e5643f8ecc1beb3a25c7 Mon Sep 17 00:00:00 2001
From: Tom Benson <benson31@llnl.gov>
Date: Wed, 4 Sep 2024 18:16:30 -0400
Subject: [PATCH] Make the separate communication stream optional (#184)

If this option is not set, the communication will land on the
Hydrogen-default stream. This _could_ be a separate stream from the
data compute stream, but in the LBANN use case, it generally will be
the compute stream.
---
 include/El/core/imports/aluminum.hpp | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/include/El/core/imports/aluminum.hpp b/include/El/core/imports/aluminum.hpp
index 911ead25d..f05092893 100644
--- a/include/El/core/imports/aluminum.hpp
+++ b/include/El/core/imports/aluminum.hpp
@@ -359,10 +359,25 @@ struct SyncInfoManager<Device::GPU>
 };
 #endif // HYDROGEN_HAVE_GPU
 
+inline bool use_separate_comm_stream() noexcept
+{
+    char const* const env = std::getenv("H_USE_SEPARATE_COMM_STREAM");
+    return (env && std::strlen(env) && env[0] != '0');
+}
+
 template <typename BackendT>
 SyncInfo<DeviceForBackend<BackendT>()> const& BackendSyncInfo()
 {
     constexpr Device D = DeviceForBackend<BackendT>();
+    if constexpr (D == El::Device::GPU)
+    {
+        static bool const use_separate_stream = use_separate_comm_stream();
+        if (!use_separate_stream)
+        {
+            return El::gpu::DefaultSyncInfo();
+        }
+    }
+
     static SyncInfoManager<D> si_mgr_(BackendT::Name());
     return si_mgr_.si_;
 }