From c56c900037ab93e5014355f0d86941e437ddd7cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Bl=C3=BChdorn?= Date: Thu, 18 Jan 2024 16:35:49 +0100 Subject: [PATCH 1/5] Add reverse atomic interface. --- .../codi/tools/parallel/atomicInterface.hpp | 9 ++- .../codi/tools/parallel/parallelToolbox.hpp | 12 ++- .../tools/parallel/reverseAtomicInterface.hpp | 80 +++++++++++++++++++ 3 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 include/codi/tools/parallel/reverseAtomicInterface.hpp diff --git a/include/codi/tools/parallel/atomicInterface.hpp b/include/codi/tools/parallel/atomicInterface.hpp index 60f8b840..2b7d5dce 100644 --- a/include/codi/tools/parallel/atomicInterface.hpp +++ b/include/codi/tools/parallel/atomicInterface.hpp @@ -40,11 +40,14 @@ namespace codi { /** - * @brief Provides a data type on which operations are performed atomically. + * @brief Provides a data type on which all operations are performed atomically. + * + * Provides also increment and decrement operators for the use case of an underlying integer type. The increment and + * decrement operators don't have to be implemented for underlying non-integer types. * * If used with an underlying floating point type or an active CoDiPack type, this data type is suitable as an adjoint - * variable type. Provides also increment and decrement operators for the use case of an underlying integer type. The - * increment and decrement operators don't have to be implemented for underlying non-integer types. + * variable type. Note, however, that it is not optimal because it performs all operations atomically. Data types + * derived from ReverseAtomicInterface are preferred as adjoint variable types. * * Implementations likely require template specializations with respect to the underlying type, especially if it is an * active CoDiPack type. diff --git a/include/codi/tools/parallel/parallelToolbox.hpp b/include/codi/tools/parallel/parallelToolbox.hpp index 7589ebf6..f211a2af 100644 --- a/include/codi/tools/parallel/parallelToolbox.hpp +++ b/include/codi/tools/parallel/parallelToolbox.hpp @@ -37,6 +37,7 @@ #include "atomicInterface.hpp" #include "mutexInterface.hpp" #include "readWriteMutex.hpp" +#include "reverseAtomicInterface.hpp" #include "staticThreadLocalPointerInterface.hpp" #include "synchronizationInterface.hpp" #include "threadInformationInterface.hpp" @@ -57,21 +58,26 @@ namespace codi { * * @tparam T_ThreadInformation Thread information used by the toolbox. See codi::ThreadInformationInterface. * @tparam T_Atomic Atomic implementation used by the toolbox. See codi::AtomicInterface. + * @tparam T_ReverseAtomic Reverse atomic implementation used by the toolbox. See + * codi::ReverseAtomicInterface. * @tparam T_Mutex Mutex implementation used by the toolbox. See codi::MutexInterface. * @tparam T_StaticThreadLocalPointer Static thread-local pointer implementation used by the toolbox. * See codi::StaticThreadLocalPointerInterface. * @tparam T_Synchronization Synchronization facalities that comply with the toolbox. * See codi::SynchronizationInterface. */ - template class T_Atomic, typename T_Mutex, - template class T_StaticThreadLocalPointer, typename T_Synchronization> + template class T_Atomic, template class T_ReverseAtomic, + typename T_Mutex, template class T_StaticThreadLocalPointer, typename T_Synchronization> struct ParallelToolbox { public: /// See codi::ParallelToolbox. using ThreadInformation = CODI_DD(T_ThreadInformation, ThreadInformationInterface); template using Atomic = CODI_DD(T_Atomic, CODI_DEFAULT_ATOMIC); ///< See codi::ParallelToolbox. - using Mutex = CODI_DD(T_Mutex, MutexInterface); ///< See codi::ParallelToolbox. + /// See codi::ParallelToolbox. + template + using ReverseAtomic = CODI_DD(T_ReverseAtomic, CODI_DEFAULT_REVERSE_ATOMIC); + using Mutex = CODI_DD(T_Mutex, MutexInterface); ///< See codi::ParallelToolbox. /// See codi::StaticThreadLocalPointerInterface. template diff --git a/include/codi/tools/parallel/reverseAtomicInterface.hpp b/include/codi/tools/parallel/reverseAtomicInterface.hpp new file mode 100644 index 00000000..ba3f09b6 --- /dev/null +++ b/include/codi/tools/parallel/reverseAtomicInterface.hpp @@ -0,0 +1,80 @@ +/* + * CoDiPack, a Code Differentiation Package + * + * Copyright (C) 2015-2024 Chair for Scientific Computing (SciComp), University of Kaiserslautern-Landau + * Homepage: http://www.scicomp.uni-kl.de + * Contact: Prof. Nicolas R. Gauger (codi@scicomp.uni-kl.de) + * + * Lead developers: Max Sagebaum, Johannes Blühdorn (SciComp, University of Kaiserslautern-Landau) + * + * This file is part of CoDiPack (http://www.scicomp.uni-kl.de/software/codi). + * + * CoDiPack is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * CoDiPack is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * + * See the GNU General Public License for more details. + * You should have received a copy of the GNU + * General Public License along with CoDiPack. + * If not, see . + * + * For other licensing options please contact us. + * + * Authors: + * - SciComp, University of Kaiserslautern-Landau: + * - Max Sagebaum + * - Johannes Blühdorn + * - Former members: + * - Tim Albring + */ +#pragma once + +#include "../../misc/macros.hpp" + +/** \copydoc codi::Namespace */ +namespace codi { + + /** + * @brief Provides a data type on which update operations are performed atomically. + * + * If used with an underlying floating point type or an active CoDiPack type, this data type is suitable as an adjoint + * variable type, as adjoint variable types only need atomic updates in a multithreaded environment. + * + * Implementations likely require template specializations with respect to the underlying type, especially if it is an + * active CoDiPack type. + * + * @tparam T_Type The underlying data type. + * @tparam T_Impl Implementing class. + */ + template + struct ReverseAtomicInterface { + public: + using Type = T_Type; ///< See ReverseAtomicInterface. + using Impl = T_Impl; ///< See ReverseAtomicInterface. + + CODI_INLINE ReverseAtomicInterface() {} ///< Constructor + CODI_INLINE ReverseAtomicInterface(ReverseAtomicInterface const&) {} ///< Constructor + CODI_INLINE ReverseAtomicInterface(Type const&) {} ///< Constructor + ~ReverseAtomicInterface() {} ///< Destructor + + /// Assignment operator with implementing type as rhs. Not atomic. + CODI_INLINE Impl& operator=(Impl const& other); + CODI_INLINE Impl& operator=(Type const& other); ///< Assignment operator with underlying type as rhs. Not atomic. + + CODI_INLINE Type operator+=(Impl const& other); ///< Atomic incremental update with implementing type as rhs. + CODI_INLINE Type operator+=(Type const& other); ///< Atomic incremental update with underlying type as rhs. + + CODI_INLINE operator Type() const; ///< Implicit cast to underlying type for rhs access. Not atomic. + }; + +#if CODI_IDE + /// Helper for IDE code completion. + template + using CODI_DEFAULT_REVERSE_ATOMIC = ReverseAtomicInterface; +#endif +} From 9c3bcdb9e3a7b2f49aca95582d941d3706bfc370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Bl=C3=BChdorn?= Date: Thu, 18 Jan 2024 16:36:57 +0100 Subject: [PATCH 2/5] OpenMP implementation. --- .../codi/tools/parallel/openmp/codiOpenMP.hpp | 3 +- .../parallel/openmp/openMPReverseAtomic.hpp | 193 ++++++++++++++++++ 2 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 include/codi/tools/parallel/openmp/openMPReverseAtomic.hpp diff --git a/include/codi/tools/parallel/openmp/codiOpenMP.hpp b/include/codi/tools/parallel/openmp/codiOpenMP.hpp index c6002245..da09647c 100644 --- a/include/codi/tools/parallel/openmp/codiOpenMP.hpp +++ b/include/codi/tools/parallel/openmp/codiOpenMP.hpp @@ -41,6 +41,7 @@ #include "../../data/direction.hpp" #include "openMPAtomic.hpp" #include "openMPMutex.hpp" +#include "openMPReverseAtomic.hpp" #include "openMPStaticThreadLocalPointer.hpp" #include "openMPSynchronization.hpp" #include "openMPThreadInformation.hpp" @@ -49,7 +50,7 @@ namespace codi { /// Parallel toolbox for OpenMP. - using OpenMPToolbox = ParallelToolbox; /// Thread-safe external function helper for external functions jointly worked on by multiple OpenMP threads. diff --git a/include/codi/tools/parallel/openmp/openMPReverseAtomic.hpp b/include/codi/tools/parallel/openmp/openMPReverseAtomic.hpp new file mode 100644 index 00000000..dcdfdbab --- /dev/null +++ b/include/codi/tools/parallel/openmp/openMPReverseAtomic.hpp @@ -0,0 +1,193 @@ +/* + * CoDiPack, a Code Differentiation Package + * + * Copyright (C) 2015-2024 Chair for Scientific Computing (SciComp), University of Kaiserslautern-Landau + * Homepage: http://www.scicomp.uni-kl.de + * Contact: Prof. Nicolas R. Gauger (codi@scicomp.uni-kl.de) + * + * Lead developers: Max Sagebaum, Johannes Blühdorn (SciComp, University of Kaiserslautern-Landau) + * + * This file is part of CoDiPack (http://www.scicomp.uni-kl.de/software/codi). + * + * CoDiPack is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * CoDiPack is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * + * See the GNU General Public License for more details. + * You should have received a copy of the GNU + * General Public License along with CoDiPack. + * If not, see . + * + * For other licensing options please contact us. + * + * Authors: + * - SciComp, University of Kaiserslautern-Landau: + * - Max Sagebaum + * - Johannes Blühdorn + * - Former members: + * - Tim Albring + */ +#pragma once + +#include + +#include "../../../expressions/activeType.hpp" +#include "../../../traits/atomicTraits.hpp" +#include "../../../traits/realTraits.hpp" +#include "../../../traits/tapeTraits.hpp" +#include "../reverseAtomicInterface.hpp" +#include "macros.hpp" + +/** \copydoc codi::Namespace */ +namespace codi { + + /** + * @brief Reverse atomic implementation for OpenMP. + * + * OpenMP reverse atomics are disabled for all types by default. Reverse atomics for arithmetic types and forward + * CoDiPack types are enabled by specializations. + * + * See also ReverseAtomicInterface. + * + * @tparam T_Type The underlying data type. + * @tparam T_Sfinae Additional SFNIAE parameter for enable-if constructs. + */ + template + struct OpenMPReverseAtomicImpl : public ReverseAtomicInterface> { + public: + using Type = T_Type; ///< See OpenMPReverseAtomicImpl. + + OpenMPReverseAtomicImpl() = delete; ///< Constructor is deleted, will throw errors for unspecialized + ///< instantiations. + }; + +#ifndef DOXYGEN_DISABLE + + // Specialization for arithmetic types. + template + struct OpenMPReverseAtomicImpl::value>::type> + : public ReverseAtomicInterface< + T_Type, OpenMPReverseAtomicImpl::value>::type>> { + public: + using Type = T_Type; + using Base = ReverseAtomicInterface< + T_Type, OpenMPReverseAtomicImpl::value>::type>>; + + private: + Type value; + + public: + CODI_INLINE OpenMPReverseAtomicImpl() : Base(), value() {} + + CODI_INLINE OpenMPReverseAtomicImpl(OpenMPReverseAtomicImpl const& other) : Base(), value(other.value) {} + + CODI_INLINE OpenMPReverseAtomicImpl(Type const& other) : Base(), value(other) {} + + CODI_INLINE OpenMPReverseAtomicImpl& operator=(OpenMPReverseAtomicImpl const& other) { + return operator=(other.value); + } + + CODI_INLINE OpenMPReverseAtomicImpl& operator=(Type const& other) { + this->value = other; + return *this; + } + + CODI_INLINE Type operator+=(OpenMPReverseAtomicImpl const& other) { + return operator+=(other.value); + } + + CODI_INLINE Type operator+=(Type const& other) { + Type result; + CODI_OMP_ATOMIC(capture) { + this->value += other; + result = this->value; + } + return result; + } + + CODI_INLINE operator Type() const { + return value; + } + }; + + // Specialization for forward CoDiPack types. Acts on value and gradient with individual atomic operations. + template + struct OpenMPReverseAtomicImpl> + : public ReverseAtomicInterface< + T_Type, OpenMPReverseAtomicImpl>>, + public T_Type { + public: + using Type = CODI_DD(T_Type, CODI_DEFAULT_LHS_EXPRESSION); + using Base = ReverseAtomicInterface< + T_Type, OpenMPReverseAtomicImpl>>; + using Tape = typename Type::Tape; + using Real = typename Type::Real; + using Gradient = typename Type::Gradient; + + CODI_INLINE OpenMPReverseAtomicImpl() : Base(), Type() {} + + CODI_INLINE OpenMPReverseAtomicImpl(OpenMPReverseAtomicImpl const& other) : Base(), Type(other) {} + + CODI_INLINE OpenMPReverseAtomicImpl(Type const& other) : Base(), Type(other) {} + + CODI_INLINE OpenMPReverseAtomicImpl& operator=(OpenMPReverseAtomicImpl const& other) { + return operator=(static_cast(other)); + } + + CODI_INLINE OpenMPReverseAtomicImpl& operator=(Type const& other) { + Type::operator=(other); + return *this; + } + + CODI_INLINE OpenMPReverseAtomicImpl& operator+=(OpenMPReverseAtomicImpl const& other) { + return operator+=(static_cast(other)); + } + + CODI_INLINE OpenMPReverseAtomicImpl& operator+=(Type const& other) { + OpenMPReverseAtomicImpl* atomicValue = reinterpret_cast*>(&this->value()); + OpenMPReverseAtomicImpl* atomicGradient = + reinterpret_cast*>(&this->gradient()); + + *atomicValue += other.value(); + *atomicGradient += other.gradient(); + return *this; + } + + CODI_INLINE operator Type() const { + return static_cast(*this); + } + }; + +#endif + + /// Wrapper for reverse atomics for OpenMP. + /// @tparam Type An arithmetic type or CoDiPack forward type. + template + using OpenMPReverseAtomic = OpenMPReverseAtomicImpl; + + /// Declare OpenMPAtomic to be atomic in terms of AtomicTraits. + template + struct AtomicTraits::IsAtomic> : std::true_type {}; + +#ifndef DOXYGEN_DISABLE + // Specialize IsTotalZero for OpenMPReverseAtomic on arithmetic types. + template + struct RealTraits::IsTotalZero< + OpenMPReverseAtomicImpl::value>::type>> { + public: + + using Type = CODI_DD( + CODI_T(OpenMPReverseAtomicImpl::value>::type>), + OpenMPReverseAtomic); + + static CODI_INLINE bool isTotalZero(Type const& v) { + return typename Type::Type() == v; + } + }; +#endif +} From f9f04067ebfb92bce9947902fc0c1d47cbba80f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Bl=C3=BChdorn?= Date: Fri, 19 Jan 2024 11:42:30 +0100 Subject: [PATCH 3/5] Adapt OpenMP types and OpDiLib tool. --- include/codi/tools/parallel/openmp/codiOpDiLibTool.hpp | 4 ++-- include/codi/tools/parallel/openmp/codiOpenMP.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/codi/tools/parallel/openmp/codiOpDiLibTool.hpp b/include/codi/tools/parallel/openmp/codiOpDiLibTool.hpp index 22ad824a..81a57bac 100644 --- a/include/codi/tools/parallel/openmp/codiOpDiLibTool.hpp +++ b/include/codi/tools/parallel/openmp/codiOpDiLibTool.hpp @@ -43,7 +43,7 @@ #include "../../../tapes/interfaces/editingTapeInterface.hpp" #include "../../../tapes/misc/vectorAccessInterface.hpp" #include "../../../traits/atomicTraits.hpp" -#include "openMPAtomic.hpp" +#include "openMPReverseAtomic.hpp" #ifndef DOXYGEN_DISABLE @@ -167,7 +167,7 @@ struct CoDiOpDiLibTool : public opdi::ToolInterface { typename Tape::Gradient* adjoints = &tape->gradient(0); using NonAtomicGradient = codi::AtomicTraits::RemoveAtomic; - using AtomicGradient = codi::OpenMPAtomic; + using AtomicGradient = codi::OpenMPReverseAtomic; if (useAtomics) { AtomicGradient* safeAdjoints = (AtomicGradient*)adjoints; diff --git a/include/codi/tools/parallel/openmp/codiOpenMP.hpp b/include/codi/tools/parallel/openmp/codiOpenMP.hpp index da09647c..0d7c4d07 100644 --- a/include/codi/tools/parallel/openmp/codiOpenMP.hpp +++ b/include/codi/tools/parallel/openmp/codiOpenMP.hpp @@ -63,7 +63,7 @@ namespace codi { /// \copydoc codi::RealReverseIndexGen

/// This a thread-safe implementation for use with OpenMP. See \ref Example_23_OpenMP_Parallel_Codes for an example. - template, + template, typename IndexManager = ParallelReuseIndexManager> using RealReverseIndexOpenMPGen = ParallelActiveType< JacobianReuseTape>, @@ -74,5 +74,5 @@ namespace codi { /// \copydoc codi::RealReverseIndexOpenMPGen template - using RealReverseIndexVecOpenMP = RealReverseIndexOpenMPGen, dim>>; + using RealReverseIndexVecOpenMP = RealReverseIndexOpenMPGen, dim>>; } From 8d6df796cf56390d3a336580f803c029c4397acb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Bl=C3=BChdorn?= Date: Fri, 19 Jan 2024 11:42:47 +0100 Subject: [PATCH 4/5] Eliminate return types. --- .../parallel/openmp/openMPReverseAtomic.hpp | 21 +++++++------------ .../tools/parallel/reverseAtomicInterface.hpp | 4 ++-- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/include/codi/tools/parallel/openmp/openMPReverseAtomic.hpp b/include/codi/tools/parallel/openmp/openMPReverseAtomic.hpp index dcdfdbab..8941d034 100644 --- a/include/codi/tools/parallel/openmp/openMPReverseAtomic.hpp +++ b/include/codi/tools/parallel/openmp/openMPReverseAtomic.hpp @@ -97,17 +97,13 @@ namespace codi { return *this; } - CODI_INLINE Type operator+=(OpenMPReverseAtomicImpl const& other) { - return operator+=(other.value); + CODI_INLINE void operator+=(OpenMPReverseAtomicImpl const& other) { + operator+=(other.value); } - CODI_INLINE Type operator+=(Type const& other) { - Type result; - CODI_OMP_ATOMIC(capture) { - this->value += other; - result = this->value; - } - return result; + CODI_INLINE void operator+=(Type const& other) { + CODI_OMP_ATOMIC(update) + this->value += other; } CODI_INLINE operator Type() const { @@ -144,18 +140,17 @@ namespace codi { return *this; } - CODI_INLINE OpenMPReverseAtomicImpl& operator+=(OpenMPReverseAtomicImpl const& other) { - return operator+=(static_cast(other)); + CODI_INLINE void operator+=(OpenMPReverseAtomicImpl const& other) { + operator+=(static_cast(other)); } - CODI_INLINE OpenMPReverseAtomicImpl& operator+=(Type const& other) { + CODI_INLINE void operator+=(Type const& other) { OpenMPReverseAtomicImpl* atomicValue = reinterpret_cast*>(&this->value()); OpenMPReverseAtomicImpl* atomicGradient = reinterpret_cast*>(&this->gradient()); *atomicValue += other.value(); *atomicGradient += other.gradient(); - return *this; } CODI_INLINE operator Type() const { diff --git a/include/codi/tools/parallel/reverseAtomicInterface.hpp b/include/codi/tools/parallel/reverseAtomicInterface.hpp index ba3f09b6..3590098f 100644 --- a/include/codi/tools/parallel/reverseAtomicInterface.hpp +++ b/include/codi/tools/parallel/reverseAtomicInterface.hpp @@ -66,8 +66,8 @@ namespace codi { CODI_INLINE Impl& operator=(Impl const& other); CODI_INLINE Impl& operator=(Type const& other); ///< Assignment operator with underlying type as rhs. Not atomic. - CODI_INLINE Type operator+=(Impl const& other); ///< Atomic incremental update with implementing type as rhs. - CODI_INLINE Type operator+=(Type const& other); ///< Atomic incremental update with underlying type as rhs. + CODI_INLINE void operator+=(Impl const& other); ///< Atomic incremental update with implementing type as rhs. + CODI_INLINE void operator+=(Type const& other); ///< Atomic incremental update with underlying type as rhs. CODI_INLINE operator Type() const; ///< Implicit cast to underlying type for rhs access. Not atomic. }; From 33257c419460e28416631d3cbf9e2bb8b3d789b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Bl=C3=BChdorn?= Date: Mon, 26 Feb 2024 11:36:09 +0100 Subject: [PATCH 5/5] Update documentation. --- include/codi/tools/parallel/atomicInterface.hpp | 3 +++ .../codi/tools/parallel/reverseAtomicInterface.hpp | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/codi/tools/parallel/atomicInterface.hpp b/include/codi/tools/parallel/atomicInterface.hpp index 2b7d5dce..33665bf1 100644 --- a/include/codi/tools/parallel/atomicInterface.hpp +++ b/include/codi/tools/parallel/atomicInterface.hpp @@ -52,6 +52,9 @@ namespace codi { * Implementations likely require template specializations with respect to the underlying type, especially if it is an * active CoDiPack type. * + * An implementation should preserve the memory footprint of the underlying type, e.g., by inheriting from the under- + * lying type or by having a variable of the underlying type as the only member variable. + * * @tparam T_Type The underlying data type. * @tparam T_Impl Implementing class. */ diff --git a/include/codi/tools/parallel/reverseAtomicInterface.hpp b/include/codi/tools/parallel/reverseAtomicInterface.hpp index 3590098f..b042e055 100644 --- a/include/codi/tools/parallel/reverseAtomicInterface.hpp +++ b/include/codi/tools/parallel/reverseAtomicInterface.hpp @@ -40,14 +40,19 @@ namespace codi { /** - * @brief Provides a data type on which update operations are performed atomically. + * @brief Provides a data type on which += update operations are performed atomically. * - * If used with an underlying floating point type or an active CoDiPack type, this data type is suitable as an adjoint - * variable type, as adjoint variable types only need atomic updates in a multithreaded environment. + * In a multithreaded environment, data races on adjoint variables are fixed by performing updates atomically, whereas + * other read and write operations do not need to be atomic. This template takes an ordinary adjoint variable type, + * like a floating point type or a CoDiPack forward type, and ensures that the corresponding += update operation is + * performed atomically. * * Implementations likely require template specializations with respect to the underlying type, especially if it is an * active CoDiPack type. * + * An implementation should preserve the memory footprint of the underlying type, e.g., by inheriting from the under- + * lying type or by having a variable of the underlying type as the only member variable. + * * @tparam T_Type The underlying data type. * @tparam T_Impl Implementing class. */