diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp index fdf679879ff2c8..56d3f31b5acbf6 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp @@ -210,6 +210,7 @@ REGISTER_FACTORY(v6, ExperimentalDetectronROIFeatureExtractor); // ------------------------------ Supported v7 ops ------------------------------ // REGISTER_FACTORY(v7, Gather); +REGISTER_FACTORY(v7, Gelu); // ------------------------------ Supported v8 ops ------------------------------ // REGISTER_FACTORY(v8, Slice); diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/activation.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/activation.hpp index ce0cbc4f829c82..ab928f128f1e70 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/activation.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/activation.hpp @@ -60,6 +60,7 @@ enum class activation_func { hswish, // val * min(max(0, val + 3), 6) / 6 mish, // val*tanh(ln(1 + exp(val))) gelu, // (0.5*val*(1 + erf(val / sqrt(2))) + gelu_tanh, // x⋅0.5⋅(1+tanh[sqrt(2/pi)*val(1 + 0.044715⋅val^2)]) round_half_to_even, // round halfs to the nearest even integer round_half_away_from_zero // round the number so it's further away from zero }; diff --git a/src/plugins/intel_gpu/src/graph/kernel_selector_helper.cpp b/src/plugins/intel_gpu/src/graph/kernel_selector_helper.cpp index 48f050e0aaba1a..e18b538fda27ad 100644 --- a/src/plugins/intel_gpu/src/graph/kernel_selector_helper.cpp +++ b/src/plugins/intel_gpu/src/graph/kernel_selector_helper.cpp @@ -815,6 +815,8 @@ kernel_selector::activation_function get_kernel_selector_activation_param(activa return kernel_selector::activation_function::MISH; case cldnn::activation_func::gelu: return kernel_selector::activation_function::GELU; + case cldnn::activation_func::gelu_tanh: + return kernel_selector::activation_function::GELU_TANH; case cldnn::activation_func::round_half_to_even: return kernel_selector::activation_function::ROUND_HALF_TO_EVEN; case cldnn::activation_func::round_half_away_from_zero: diff --git a/src/plugins/intel_gpu/src/kernel_selector/common/common_types.h b/src/plugins/intel_gpu/src/kernel_selector/common/common_types.h index 505bc9335c5682..73d21d8a6559ae 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/common/common_types.h +++ b/src/plugins/intel_gpu/src/kernel_selector/common/common_types.h @@ -156,6 +156,7 @@ enum class ActivationFunction { HSWISH, MISH, GELU, + GELU_TANH, ROUND_HALF_TO_EVEN, ROUND_HALF_AWAY_FROM_ZERO }; diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/common/jitter.cpp b/src/plugins/intel_gpu/src/kernel_selector/core/common/jitter.cpp index 120c4763a16692..2a29ac9ec6d043 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/common/jitter.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/core/common/jitter.cpp @@ -91,6 +91,11 @@ JitTerm erf(const JitTerm& arg) { return jit_term; } +JitTerm tanh(const JitTerm& arg) { + JitTerm jit_term{"(tanh(" + arg.str() + "))"}; + return jit_term; +} + JitTerm log(const JitTerm& arg) { JitTerm jit_term{"(log(" + arg.str() + "))"}; return jit_term; @@ -1116,6 +1121,16 @@ JitConstants MakeActivationJitConstants(ActivationFunction activation_function, (half * input * (one + erf((input * mult)))).str())); break; } + case ActivationFunction::GELU_TANH: { + const std::string type_suffix = out_dt == Datatype::F32 ? "f" : "h"; + const JitTerm half{"0.5" + type_suffix}; + const JitTerm mult{"0.044715" + type_suffix}; + const JitTerm sqrt_2_over_pi{"0.79788458347320556640625" + type_suffix}; + jitConstants.AddConstant(MakeJitConstant( + macro_def, + (half * input * (one + tanh(sqrt_2_over_pi * input * (one + mult * input * input)))).str())); + break; + } case ActivationFunction::NOT: jitConstants.AddConstant(MakeJitConstant( macro_def, diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/kernel_selector_common.cpp b/src/plugins/intel_gpu/src/kernel_selector/core/kernel_selector_common.cpp index 79a6c0e0b2958b..29a387d125d795 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/kernel_selector_common.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/core/kernel_selector_common.cpp @@ -75,6 +75,7 @@ std::string toString(ActivationFunction activation) { case ActivationFunction::HSWISH: method = "HSWISH"; break; case ActivationFunction::MISH: method = "MISH"; break; case ActivationFunction::GELU: method = "GELU"; break; + case ActivationFunction::GELU_TANH: method = "GELU_TANH"; break; case ActivationFunction::ROUND_HALF_TO_EVEN: method = "ROUND_HALF_TO_EVEN"; break; case ActivationFunction::ROUND_HALF_AWAY_FROM_ZERO: method = "ROUND_HALF_AWAY_FROM_ZERO"; break; default: break; diff --git a/src/plugins/intel_gpu/src/plugin/ops/unary.cpp b/src/plugins/intel_gpu/src/plugin/ops/unary.cpp index 232b13a9ad5950..251e10572823df 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/unary.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/unary.cpp @@ -256,8 +256,15 @@ static void CreateMishOp(Program& p, const std::shared_ptr CreateUnaryEltwiseOp(p, op, cldnn::activation_func::mish, {}); } -static void CreateGeluOp(Program& p, const std::shared_ptr& op) { - CreateUnaryEltwiseOp(p, op, cldnn::activation_func::gelu, {}); +static void CreateGeluOp(Program& p, const std::shared_ptr& op) { + cldnn::activation_func activationFunc = + op->get_approximation_mode() == op::GeluApproximationMode::ERF ? cldnn::activation_func::gelu + : cldnn::activation_func::gelu_tanh; + CreateUnaryEltwiseOp(p, op, activationFunc, {}); +} + +static void CreateGeluOp(Program &p, const std::shared_ptr& op) { + CreateUnaryEltwiseOp(p, op, cldnn::activation_func::gelu, {}); } static void CreateSignOp(Program& p, const std::shared_ptr& op) { @@ -311,6 +318,7 @@ REGISTER_FACTORY_IMPL(v4, Swish); REGISTER_FACTORY_IMPL(v4, HSwish); REGISTER_FACTORY_IMPL(v4, Mish); REGISTER_FACTORY_IMPL(v0, Gelu); +REGISTER_FACTORY_IMPL(v7, Gelu); REGISTER_FACTORY_IMPL(v0, Sign); REGISTER_FACTORY_IMPL(v5, HSigmoid); REGISTER_FACTORY_IMPL(v5, Round); diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 4d72d2f68cc6b9..1454ac4f50adb5 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -71,6 +71,7 @@ #include #include #include "transformations/op_conversions/softmax_decomposition.hpp" +#include #include #include #include @@ -300,6 +301,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { // List of enabled/disabled transformations pass_config->disable(); + pass_config->disable(); pass_config->disable(); pass_config->disable(); pass_config->disable(); diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/single_layer_tests/activation.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/single_layer_tests/activation.cpp index f73281c473dbda..4988a039ac6f78 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/single_layer_tests/activation.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/single_layer_tests/activation.cpp @@ -57,7 +57,9 @@ const std::map>> activationTypes {HSigmoid, {}}, {Swish, {{0.5f}}}, {RoundHalfToEven, {}}, - {RoundHalfAwayFromZero, {}} + {RoundHalfAwayFromZero, {}}, + {GeluErf, {}}, + {GeluTanh, {}} }; const std::map>> activationParamTypes = {