diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
index 6b9fe68b699db90..bb0c018dc36f364 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
@@ -341,7 +341,9 @@ protected void runWaitingTxnJob() throws AlterCancelException {
 
             LOG.info("invertedIndexBatchTask:{}", invertedIndexBatchTask);
             AgentTaskQueue.addBatchTask(invertedIndexBatchTask);
-            AgentTaskExecutor.submit(invertedIndexBatchTask);
+            if (!FeConstants.runningUnitTest) {
+                AgentTaskExecutor.submit(invertedIndexBatchTask);
+            }
         } finally {
             olapTable.readUnlock();
         }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
index 120fec4ab03b639..5a4ce9347d4e310 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
@@ -491,6 +491,7 @@ public void testBuildIndexFailedWithMinFailedNum() throws Exception {
         // if task error is not OBTAIN_LOCK_FAILED, the job should be failed after MIN_FAILED_NUM = 3 times
         AgentTask agentTask = tasks.get(0);
         agentTask.setErrorCode(TStatusCode.IO_ERROR);
+        Assert.assertEquals(agentTask.getFailedTimes(), 0);
         for (int i = 0; i < IndexChangeJob.MIN_FAILED_NUM; i++) {
             agentTask.failed();
             schemaChangeHandler.runAfterCatalogReady();
@@ -541,6 +542,7 @@ public void testBuildIndexFailedWithMaxFailedNum() throws Exception {
         // if task error is OBTAIN_LOCK_FAILED, the job should be failed after MAX_FAILED_NUM = 10 times
         AgentTask agentTask = tasks.get(0);
         agentTask.setErrorCode(TStatusCode.OBTAIN_LOCK_FAILED);
+        Assert.assertEquals(agentTask.getFailedTimes(), 0);
         for (int i = 0; i < IndexChangeJob.MAX_FAILED_NUM; i++) {
             agentTask.failed();
             schemaChangeHandler.runAfterCatalogReady();