From b1c8364d7b29c2eaa01d321c0dfc1ffa4d525bd6 Mon Sep 17 00:00:00 2001 From: sawyersong2 Date: Wed, 28 Aug 2024 15:54:37 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat=EF=BC=9A=E4=BC=98=E5=8C=96dispatch-sdk?= =?UTF-8?q?=E8=B0=83=E5=BA=A6=E9=80=BB=E8=BE=91=E5=AF=B9=E5=85=B6=E4=BB=96?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E7=9A=84=E4=BE=9D=E8=B5=96=20#10882?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dispatch.sdk/service/DispatchService.kt | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt index 2a5d734fb86..e1462ce0f03 100644 --- a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt +++ b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt @@ -64,6 +64,7 @@ import com.tencent.devops.process.engine.pojo.PipelineBuildContainer import com.tencent.devops.process.engine.pojo.PipelineBuildTask import com.tencent.devops.process.pojo.mq.PipelineAgentShutdownEvent import com.tencent.devops.process.pojo.mq.PipelineAgentStartupEvent +import feign.RetryableException import java.util.Date import java.util.concurrent.TimeUnit import org.slf4j.LoggerFactory @@ -153,7 +154,7 @@ class DispatchService constructor( } fun checkRunning(event: PipelineAgentStartupEvent): Boolean { - val (startBuildTask, buildContainer) = getContainerStartupInfo(event) + val (startBuildTask, buildContainer) = getContainerStartupInfoWithRetry(event) var needStart = true if (event.executeCount != startBuildTask.executeCount) { @@ -187,7 +188,7 @@ class DispatchService constructor( fun onContainerFailure(event: PipelineAgentStartupEvent, e: BuildFailureException) { logger.warn("[${event.buildId}|${event.vmSeqId}] Container startup failure") try { - val (startBuildTask, buildContainer) = getContainerStartupInfo(event) + val (startBuildTask, buildContainer) = getContainerStartupInfoWithRetry(event) if (buildContainer.status.isCancel() || startBuildTask.status.isCancel()) { return } @@ -249,6 +250,26 @@ class DispatchService constructor( } } + private fun getContainerStartupInfoWithRetry( + event: PipelineAgentStartupEvent, + retryTimes: Int = RETRY_TIMES + ): Pair { + with(event) { + try { + return getContainerStartupInfo(event) + } catch (e: RetryableException) { + if (retryTimes > 0) { + logger.warn("[$pipelineId]|[$buildId]|[$vmSeqId]| getContainerStartupInfo failed, " + + "retryTimes=$retryTimes", e.message) + Thread.sleep(1000) + return getContainerStartupInfoWithRetry(event, retryTimes - 1) + } else { + throw e + } + } + } + } + private fun getContainerStartupInfo( event: PipelineAgentStartupEvent ): Pair { @@ -344,5 +365,7 @@ class DispatchService constructor( companion object { private val logger = LoggerFactory.getLogger(DispatchService::class.java) + + private const val RETRY_TIMES = 3 } } From 05902709fa9c9569aa12d69f605f15a0513de8d5 Mon Sep 17 00:00:00 2001 From: sawyersong2 Date: Wed, 28 Aug 2024 15:56:03 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat=EF=BC=9A=E4=BC=98=E5=8C=96dispatch-sdk?= =?UTF-8?q?=E8=B0=83=E5=BA=A6=E9=80=BB=E8=BE=91=E5=AF=B9=E5=85=B6=E4=BB=96?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E7=9A=84=E4=BE=9D=E8=B5=96=20#10882?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/common/dispatch.sdk/service/DispatchService.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt index e1462ce0f03..a7e77fdc824 100644 --- a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt +++ b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt @@ -250,6 +250,9 @@ class DispatchService constructor( } } + /** + * 针对服务间调用出现 Connection refused 的异常，进行重试 + */ private fun getContainerStartupInfoWithRetry( event: PipelineAgentStartupEvent, retryTimes: Int = RETRY_TIMES From 05b60a2a10b3194d93950f1c37ed3f1d4076c654 Mon Sep 17 00:00:00 2001 From: sawyersong2 Date: Wed, 20 Nov 2024 14:46:00 +0800 Subject: [PATCH 3/3] =?UTF-8?q?feat=EF=BC=9A=E4=BC=98=E5=8C=96dispatch-sdk?= =?UTF-8?q?=E8=B0=83=E5=BA=A6=E9=80=BB=E8=BE=91=E5=AF=B9=E5=85=B6=E4=BB=96?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E7=9A=84=E4=BE=9D=E8=B5=96=20#10882?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/common/dispatch.sdk/service/DispatchService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt index 303f5199f14..385e4304c00 100644 --- a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt +++ b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt @@ -67,7 +67,7 @@ import com.tencent.devops.process.pojo.mq.PipelineAgentShutdownEvent import com.tencent.devops.process.pojo.mq.PipelineAgentStartupEvent import feign.RetryableException import org.slf4j.LoggerFactory -import java.util.* +import java.util.Date import java.util.concurrent.TimeUnit @Suppress("LongParameterList", "TooManyFunctions")