[FLINK-36198][network] Remove deprecated config options related to le…

…gacy hybrid shuffle
dufeng1010 · Sep 3, 2024 · ae1d7c1 · ae1d7c1
1 parent 5e36124
commit ae1d7c1
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 69 deletions.
diff --git a/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html b/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html
@@ -30,7 +30,7 @@
             <td><h5>taskmanager.network.hybrid-shuffle.remote.path</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
             <td>String</td>
-            <td>The option is used to configure the base path of remote storage for hybrid shuffle. The shuffle data will be stored in remote storage when the disk space is not enough. Note: If the option is configured and taskmanager.network.hybrid-shuffle.enable-new-mode is false, this option will be ignored. If the option is not configured and taskmanager.network.hybrid-shuffle.enable-new-mode is true, the remote storage will be disabled.</td>
+            <td>The option is used to configure the base path of remote storage for hybrid shuffle. The shuffle data will be stored in remote storage when the disk space is not enough. Note: If this option is not configured the remote storage will be disabled.</td>
         </tr>
         <tr>
             <td><h5>taskmanager.network.memory.buffer-debloat.enabled</h5></td>

diff --git a/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html b/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html
@@ -48,7 +48,7 @@
             <td><h5>taskmanager.network.hybrid-shuffle.remote.path</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
             <td>String</td>
-            <td>The option is used to configure the base path of remote storage for hybrid shuffle. The shuffle data will be stored in remote storage when the disk space is not enough. Note: If the option is configured and taskmanager.network.hybrid-shuffle.enable-new-mode is false, this option will be ignored. If the option is not configured and taskmanager.network.hybrid-shuffle.enable-new-mode is true, the remote storage will be disabled.</td>
+            <td>The option is used to configure the base path of remote storage for hybrid shuffle. The shuffle data will be stored in remote storage when the disk space is not enough. Note: If this option is not configured the remote storage will be disabled.</td>
         </tr>
         <tr>
             <td><h5>taskmanager.network.memory.read-buffer.required-per-gate.max</h5></td>

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java
@@ -31,10 +31,6 @@
 /** The set of configuration options relating to network stack. */
 @PublicEvolving
 public class NettyShuffleEnvironmentOptions {
-
-    private static final String HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME =
-            "taskmanager.network.hybrid-shuffle.enable-new-mode";
-
     private static final String HYBRID_SHUFFLE_REMOTE_STORAGE_BASE_PATH_OPTION_NAME =
             "taskmanager.network.hybrid-shuffle.remote.path";
 
@@ -357,44 +353,6 @@ public enum CompressionCodec {
                                     // this raw value must be changed correspondingly
                                     "taskmanager.memory.framework.off-heap.batch-shuffle.size"));
 
-    /**
-     * Region group size of hybrid spilled file data index.
-     *
-     * @deprecated The option is only used in the legacy hybrid shuffle mode. It is deprecated in
-     *     1.20 and will be totally removed in 2.0, as the legacy hybrid shuffle mode will be
-     *     removed in 2.0.
-     */
-    @Deprecated
-    public static final ConfigOption<Integer> HYBRID_SHUFFLE_SPILLED_INDEX_REGION_GROUP_SIZE =
-            key("taskmanager.network.hybrid-shuffle.spill-index-region-group-size")
-                    .intType()
-                    .defaultValue(1024)
-                    .withDeprecatedKeys(
-                            "taskmanager.network.hybrid-shuffle.spill-index-segment-size")
-                    .withDescription(
-                            "Controls the region group size(in bytes) of hybrid spilled file data index. "
-                                    + "Note: This option will be ignored if "
-                                    + HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME
-                                    + " is set true.");
-
-    /**
-     * Max number of hybrid retained regions in memory.
-     *
-     * @deprecated The option is only used in the legacy hybrid shuffle mode. It is deprecated in
-     *     1.20 and will be totally removed in 2.0, as the legacy hybrid shuffle mode will be
-     *     removed in 2.0.
-     */
-    @Deprecated
-    public static final ConfigOption<Long> HYBRID_SHUFFLE_NUM_RETAINED_IN_MEMORY_REGIONS_MAX =
-            key("taskmanager.network.hybrid-shuffle.num-retained-in-memory-regions-max")
-                    .longType()
-                    .defaultValue(1024 * 1024L)
-                    .withDescription(
-                            "Controls the max number of hybrid retained regions in memory. "
-                                    + "Note: This option will be ignored if "
-                                    + HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME
-                                    + " is set true. ");
-
     /**
      * Number of max buffers can be used for each output subpartition.
      *
@@ -471,25 +429,6 @@ public enum CompressionCodec {
                                     + "tasks have occupied all the buffers and the downstream tasks are waiting for the exclusive buffers. The timeout breaks"
                                     + "the tie by failing the request of exclusive buffers and ask users to increase the number of total buffers.");
 
-    /**
-     * The option to enable the new mode of hybrid shuffle.
-     *
-     * @deprecated This option is deprecated in 1.20 and will be totally removed in 2.0, as the
-     *     legacy hybrid shuffle mode will be removed in 2.0.
-     */
-    @Deprecated @Experimental
-    public static final ConfigOption<Boolean> NETWORK_HYBRID_SHUFFLE_ENABLE_NEW_MODE =
-            ConfigOptions.key(HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME)
-                    .booleanType()
-                    .defaultValue(true)
-                    .withDescription(
-                            "The option is used to enable the new mode of hybrid shuffle, which has resolved existing issues in the legacy mode. First, the new mode "
-                                    + "uses less required network memory. Second, the new mode can store shuffle data in remote storage when the disk space is not "
-                                    + "enough, which could avoid insufficient disk space errors and is only supported when "
-                                    + HYBRID_SHUFFLE_REMOTE_STORAGE_BASE_PATH_OPTION_NAME
-                                    + " is configured. The new mode is currently in an experimental phase. It can be set to false to fallback to the legacy mode "
-                                    + " if something unexpected. Once the new mode reaches a stable state, the legacy mode as well as the option will be removed.");
-
     /** The option to configure the tiered factory creator remote class name for hybrid shuffle. */
     @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
     @Experimental
@@ -518,12 +457,7 @@ public enum CompressionCodec {
                     .withDescription(
                             "The option is used to configure the base path of remote storage for hybrid shuffle. The shuffle data will be stored in "
                                     + "remote storage when the disk space is not enough. "
-                                    + "Note: If the option is configured and "
-                                    + HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME
-                                    + " is false, this option will be ignored. "
-                                    + "If the option is not configured and "
-                                    + HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME
-                                    + " is true, the remote storage will be disabled.");
+                                    + "Note: If this option is not configured the remote storage will be disabled.");
 
     /**
      * @deprecated The hash-based blocking shuffle is deprecated in 1.20 and will be totally removed