[GPU] Implement Gelu7 operation support (openvinotoolkit#9813)

* [GPU] Add Gelu tanh approximation support * [GPU] Register op7::Gelu operation in gpu plugin. * [GPU] Add op7::Gelu operation tests to GPU single layer tests
maxnick · Jan 24, 2022 · 7114253 · 7114253
1 parent ec38a88
commit 7114253
Show file tree

Hide file tree

Showing 9 changed files with 36 additions and 3 deletions.
diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp
@@ -210,6 +210,7 @@ REGISTER_FACTORY(v6, ExperimentalDetectronROIFeatureExtractor);
 
 // ------------------------------ Supported v7 ops ------------------------------ //
 REGISTER_FACTORY(v7, Gather);
+REGISTER_FACTORY(v7, Gelu);
 
 // ------------------------------ Supported v8 ops ------------------------------ //
 REGISTER_FACTORY(v8, Slice);

diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/activation.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/activation.hpp
@@ -60,6 +60,7 @@ enum class activation_func {
     hswish,                    // val * min(max(0, val + 3), 6) / 6
     mish,                      // val*tanh(ln(1 + exp(val)))
     gelu,                      // (0.5*val*(1 + erf(val / sqrt(2)))
+    gelu_tanh,                 // x⋅0.5⋅(1+tanh[sqrt(2/pi)*val(1 + 0.044715⋅val^2)])
     round_half_to_even,        // round halfs to the nearest even integer
     round_half_away_from_zero  // round the number so it's further away from zero
 };

diff --git a/src/plugins/intel_gpu/src/graph/kernel_selector_helper.cpp b/src/plugins/intel_gpu/src/graph/kernel_selector_helper.cpp
@@ -815,6 +815,8 @@ kernel_selector::activation_function get_kernel_selector_activation_param(activa
             return kernel_selector::activation_function::MISH;
         case cldnn::activation_func::gelu:
             return kernel_selector::activation_function::GELU;
+        case cldnn::activation_func::gelu_tanh:
+            return kernel_selector::activation_function::GELU_TANH;
         case cldnn::activation_func::round_half_to_even:
             return kernel_selector::activation_function::ROUND_HALF_TO_EVEN;
         case cldnn::activation_func::round_half_away_from_zero:

diff --git a/src/plugins/intel_gpu/src/kernel_selector/common/common_types.h b/src/plugins/intel_gpu/src/kernel_selector/common/common_types.h
@@ -156,6 +156,7 @@ enum class ActivationFunction {
     HSWISH,
     MISH,
     GELU,
+    GELU_TANH,
     ROUND_HALF_TO_EVEN,
     ROUND_HALF_AWAY_FROM_ZERO
 };

diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/common/jitter.cpp b/src/plugins/intel_gpu/src/kernel_selector/core/common/jitter.cpp
@@ -91,6 +91,11 @@ JitTerm erf(const JitTerm& arg) {
     return jit_term;
 }
 
+JitTerm tanh(const JitTerm& arg) {
+    JitTerm jit_term{"(tanh(" + arg.str() + "))"};
+    return jit_term;
+}
+
 JitTerm log(const JitTerm& arg) {
     JitTerm jit_term{"(log(" + arg.str() + "))"};
     return jit_term;
@@ -1116,6 +1121,16 @@ JitConstants MakeActivationJitConstants(ActivationFunction activation_function,
                     (half * input * (one + erf((input * mult)))).str()));
             break;
         }
+        case ActivationFunction::GELU_TANH: {
+            const std::string type_suffix = out_dt == Datatype::F32 ? "f" : "h";
+            const JitTerm half{"0.5" + type_suffix};
+            const JitTerm mult{"0.044715" + type_suffix};
+            const JitTerm sqrt_2_over_pi{"0.79788458347320556640625" + type_suffix};
+            jitConstants.AddConstant(MakeJitConstant(
+                    macro_def,
+                    (half * input * (one + tanh(sqrt_2_over_pi * input * (one + mult * input * input)))).str()));
+            break;
+        }
         case ActivationFunction::NOT:
             jitConstants.AddConstant(MakeJitConstant(
                 macro_def,

diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/kernel_selector_common.cpp b/src/plugins/intel_gpu/src/kernel_selector/core/kernel_selector_common.cpp
@@ -75,6 +75,7 @@ std::string toString(ActivationFunction activation) {
         case ActivationFunction::HSWISH:                    method = "HSWISH"; break;
         case ActivationFunction::MISH:                      method = "MISH"; break;
         case ActivationFunction::GELU:                      method = "GELU"; break;
+        case ActivationFunction::GELU_TANH:                 method = "GELU_TANH"; break;
         case ActivationFunction::ROUND_HALF_TO_EVEN:        method = "ROUND_HALF_TO_EVEN"; break;
         case ActivationFunction::ROUND_HALF_AWAY_FROM_ZERO: method = "ROUND_HALF_AWAY_FROM_ZERO"; break;
         default: break;

diff --git a/src/plugins/intel_gpu/src/plugin/ops/unary.cpp b/src/plugins/intel_gpu/src/plugin/ops/unary.cpp
@@ -256,8 +256,15 @@ static void CreateMishOp(Program& p, const std::shared_ptr<ngraph::op::v4::Mish>
     CreateUnaryEltwiseOp(p, op, cldnn::activation_func::mish, {});
 }
 
-static void CreateGeluOp(Program& p, const std::shared_ptr<ngraph::op::v0::Gelu>& op) {
-    CreateUnaryEltwiseOp(p, op, cldnn::activation_func::gelu, {});
+static void CreateGeluOp(Program& p, const std::shared_ptr<ngraph::op::v7::Gelu>& op) {
+    cldnn::activation_func activationFunc =
+            op->get_approximation_mode() == op::GeluApproximationMode::ERF ? cldnn::activation_func::gelu
+                                                                           : cldnn::activation_func::gelu_tanh;
+    CreateUnaryEltwiseOp(p, op, activationFunc, {});
+}
+
+static void CreateGeluOp(Program &p, const std::shared_ptr<ngraph::op::v0::Gelu>& op) {
+    CreateUnaryEltwiseOp(p, op,  cldnn::activation_func::gelu, {});
 }
 
 static void CreateSignOp(Program& p, const std::shared_ptr<ngraph::op::v0::Sign>& op) {
@@ -311,6 +318,7 @@ REGISTER_FACTORY_IMPL(v4, Swish);
 REGISTER_FACTORY_IMPL(v4, HSwish);
 REGISTER_FACTORY_IMPL(v4, Mish);
 REGISTER_FACTORY_IMPL(v0, Gelu);
+REGISTER_FACTORY_IMPL(v7, Gelu);
 REGISTER_FACTORY_IMPL(v0, Sign);
 REGISTER_FACTORY_IMPL(v5, HSigmoid);
 REGISTER_FACTORY_IMPL(v5, Round);

diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp
@@ -71,6 +71,7 @@
 #include <transformations/op_conversions/convert_deformable_conv_v8_to_v1.hpp>
 #include <transformations/op_conversions/simplify_ctc_greedy_decoder_seq_len.hpp>
 #include "transformations/op_conversions/softmax_decomposition.hpp"
+#include <transformations/op_conversions/gelu7_downgrade.hpp>
 #include <transformations/convert_precision.hpp>
 #include <transformations/init_node_info.hpp>
 #include <transformations/rt_info/fused_names_attribute.hpp>
@@ -300,6 +301,7 @@ void TransformationsPipeline::apply(std::shared_ptr<ov::Model> func) {
 
         // List of enabled/disabled transformations
         pass_config->disable<ngraph::pass::ConvertGELU>();
+        pass_config->disable<ngraph::pass::Gelu7Downgrade>();
         pass_config->disable<ngraph::pass::ConvertMod>();
         pass_config->disable<ngraph::pass::ConvertShuffleChannels3>();
         pass_config->disable<ngraph::pass::HSwishDecomposition>();

diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/single_layer_tests/activation.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/single_layer_tests/activation.cpp
@@ -57,7 +57,9 @@ const std::map<ActivationTypes, std::vector<std::vector<float>>> activationTypes
         {HSigmoid,              {}},
         {Swish,                 {{0.5f}}},
         {RoundHalfToEven,       {}},
-        {RoundHalfAwayFromZero, {}}
+        {RoundHalfAwayFromZero, {}},
+        {GeluErf,               {}},
+        {GeluTanh,              {}}
 };
 
 const std::map<ActivationTypes, std::vector<std::vector<float>>> activationParamTypes = {
-Original file line number
+Diff line change
@@ Expand Up / @@ -156,6 +156,7 @@ enum class ActivationFunction { @@
         HSWISH,
         MISH,
         GELU,
+        GELU_TANH,
         ROUND_HALF_TO_EVEN,
         ROUND_HALF_AWAY_FROM_ZERO
     };
@@ Expand Down @@