From d5c0b468d0cd304860cff73996c448530ac3f469 Mon Sep 17 00:00:00 2001 From: owenlxu Date: Fri, 25 Aug 2023 19:54:20 +0800 Subject: [PATCH 01/11] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E7=A6=BB=E5=B2=B8token=E6=A0=A1=E9=AA=8C=E5=9C=BA=E6=99=AF=20#?= =?UTF-8?q?1095?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gateway/lua/auth/auth_web.lua | 11 ++++++ src/gateway/lua/util/oauth_util.lua | 54 ++++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/gateway/lua/auth/auth_web.lua b/src/gateway/lua/auth/auth_web.lua index 1513cb0bc9..3471d42bea 100644 --- a/src/gateway/lua/auth/auth_web.lua +++ b/src/gateway/lua/auth/auth_web.lua @@ -59,6 +59,17 @@ elseif config.auth_mode == "ticket" then end username = oauthUtil:verify_ticket(bk_ticket, "ticket") token = bk_ticket +elseif config.auth_mode == "odc" then + local bk_token = cookieUtil:get_cookie("bk_token") + if bk_token == nil then + bk_token = ngx.var.http_x_devops_bk_token + if bk_token == nil then + ngx.exit(401) + return + end + end + username = oauthUtil:verify_tai_token(bk_token) + token = bk_token elseif config.auth_mode == "ci" then local ci_login_token = cookieUtil:get_cookie("X-DEVOPS-CI-LOGIN-TOKEN") if not ci_login_token then diff --git a/src/gateway/lua/util/oauth_util.lua b/src/gateway/lua/util/oauth_util.lua index 56ada0adc0..a2652f97ae 100644 --- a/src/gateway/lua/util/oauth_util.lua +++ b/src/gateway/lua/util/oauth_util.lua @@ -161,7 +161,7 @@ function _M:verify_bk_token(auth_url, token) if result.code ~= 0 then if result.code == 1302403 then ngx.log(ngx.ERR, "is_login code is 1302403 , need Authentication") - ngx.header["X-DEVOPS-ERROR-RETURN"] = '{"code": 440,"message": "'..result.message..'", "data": 1302403,"traceId":null }' + ngx.header["X-DEVOPS-ERROR-RETURN"] = '{"code": 440,"message": "' .. result.message .. '", "data": 1302403,"traceId":null }' ngx.header["X-DEVOPS-ERROR-STATUS"] = 440 ngx.exit(401) end @@ -284,4 +284,56 @@ function _M:verify_ci_token(ci_login_token) return user_cache_value end +function _M:verify_tai_token(tai_token) + local user_cache = ngx.shared.user_info_store + local user_cache_value = user_cache:get(tai_token) + if user_cache_value == nil then + local http_cli = http.new() + oauth = config.oauth + local addr = "https://" .. oauth.url .. "/prod/bk_token_check/?bk_token=" .. tai_token + local auth_content = '{"bk_app_code":"' .. oauth.app_code .. '","bk_app_secret":"' .. oauth.app_secret .. '"}' + --- 开始连接 + http_cli:set_timeout(3000) + http_cli:connect(addr) + --- 发送请求 + local res, err = http_cli:request_uri(addr, { + method = "GET", + ssl_verify = false, + headers = { + ["X-Bkapi-Authorization"] = auth_content + } + }) + --- 判断是否出错了 + if not res then + ngx.log(ngx.ERR, "failed to request tai token: error", err) + ngx.exit(401) + return + end + --- 判断返回的状态码是否是200 + if res.status ~= 200 then + ngx.log(ngx.STDERR, "failed to request tai token, status: ", res.status) + ngx.exit(401) + return + end + --- 转换JSON的返回数据为TABLE + local result = json.decode(res.body) + --- 判断JSON转换是否成功 + if result == nil then + ngx.log(ngx.ERR, "failed to parse tai token response:", res.body) + ngx.exit(401) + return + end + + --- 判断返回名字 + if result.result ~= true or result.code ~= "00" then + ngx.log(ngx.INFO, "invalid user tai token: ", result.message) + ngx.exit(401) + return + end + user_cache_value = result.data.username + user_cache:set(tai_token, user_cache_value, 180) + end + return user_cache_value +end + return _M From eee74457abc36490940ffbc452f614a08058e043 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Tue, 29 Aug 2023 11:08:55 +0800 Subject: [PATCH 02/11] =?UTF-8?q?feat:=E5=A2=9E=E5=8A=A0=E4=BB=93=E5=BA=93?= =?UTF-8?q?=E7=BA=A7=E5=88=AB=E4=B8=8B=E8=BD=BD=E9=99=90=E9=80=9F=20#1092?= =?UTF-8?q?=20(#1094)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 增加仓库级别限速 #1092 * feat: 剔除上传限制 #1092 * feat: 代码还原 #1092 * feat: 增加缓存 #1092 * feat: 当仓库配置下载限速小于等于最低限速时则直接将请求断开, 避免占用过多连接 #1092 * feat: 增加配置 #1092 * feat: 配置调整#1092 --- .../api/exception/TooManyRequestsException.kt | 38 ++++++++++++++++ .../common/api/message/CommonMessageCode.kt | 1 + .../artifact/constant/ArtifactConstants.kt | 5 +++ .../context/ArtifactContextHolder.kt | 35 +++++++++++++++ .../response/DefaultArtifactResourceWriter.kt | 35 ++++++++++++++- .../exception/GlobalExceptionHandler.kt | 6 +++ .../resources/i18n/messages_en.properties | 1 + .../resources/i18n/messages_zh_CN.properties | 1 + .../resources/i18n/messages_zh_TW.properties | 1 + .../core/config/RateLimitProperties.kt | 44 +++++++++++++++++++ .../storage/core/config/ResponseProperties.kt | 6 ++- 11 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/TooManyRequestsException.kt create mode 100644 src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/config/RateLimitProperties.kt diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/TooManyRequestsException.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/TooManyRequestsException.kt new file mode 100644 index 0000000000..05a955fb16 --- /dev/null +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/TooManyRequestsException.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.api.exception + +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.api.message.CommonMessageCode + +/** + * 请求过多,超出了频次限制 + */ +class TooManyRequestsException( + parameter: String = "Too Many Requests" +) : ErrorCodeException(HttpStatus.TOO_MANY_REQUESTS, CommonMessageCode.TOO_MANY_REQUESTS, arrayOf(parameter)) \ No newline at end of file diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt index d529f4f019..717cdec6eb 100644 --- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt @@ -55,6 +55,7 @@ enum class CommonMessageCode(private val key: String) : MessageCode { MODIFY_PASSWORD_FAILED("modify.password.failed"), OPERATION_CROSS_CLUSTER_NOT_ALLOWED("operation.cross-cluster.not-allowed"), MEDIA_TYPE_UNACCEPTABLE("system.media-type.unacceptable"), + TOO_MANY_REQUESTS("too.many.requests") ; override fun getBusinessCode() = ordinal + 1 diff --git a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt index 63ab74ca67..dc42d579cc 100644 --- a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt +++ b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt @@ -51,6 +51,11 @@ const val ARTIFACT_CONFIGURER = "artifact-configurer" */ const val NODE_DETAIL_KEY = "node-detail" +/** + * 查询仓库上传下载限速配置后后将其写入request attributes的key + */ +const val REPO_RATE_LIMIT_KEY = "repo-rate-limit" + /** * 项目id字段 */ diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt index 81f56d9c54..6fa5a17a53 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt @@ -41,6 +41,7 @@ import com.tencent.bkrepo.common.artifact.constant.NODE_DETAIL_KEY import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID import com.tencent.bkrepo.common.artifact.constant.REPO_KEY import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.common.artifact.constant.REPO_RATE_LIMIT_KEY import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.RepositoryType @@ -48,11 +49,13 @@ import com.tencent.bkrepo.common.artifact.repository.composite.CompositeReposito import com.tencent.bkrepo.common.artifact.repository.core.ArtifactRepository import com.tencent.bkrepo.common.security.http.core.HttpAuthSecurity import com.tencent.bkrepo.common.service.util.HttpContextHolder +import com.tencent.bkrepo.common.storage.core.config.RateLimitProperties import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import org.springframework.beans.factory.ObjectProvider +import org.springframework.util.unit.DataSize import org.springframework.web.servlet.HandlerMapping import java.util.concurrent.TimeUnit import javax.servlet.http.HttpServletRequest @@ -85,6 +88,10 @@ class ArtifactContextHolder( private lateinit var nodeClient: NodeClient private lateinit var httpAuthSecurity: ObjectProvider + + private const val RECEIVE_RATE_LIMIT_OF_REPO = "receiveRateLimit" + private const val RESPONSE_RATE_LIMIT_OF_REPO = "responseRateLimit" + private val artifactConfigurerMap = mutableMapOf() private val repositoryDetailCache = CacheBuilder.newBuilder() .maximumSize(1000) @@ -277,6 +284,34 @@ class ArtifactContextHolder( nodeDetail?.let { request.setAttribute(attrKey, nodeDetail) } return nodeDetail } + + /** + * 获取仓库级别的限速配置 + */ + fun getRateLimitOfRepo(): RateLimitProperties { + val request = HttpContextHolder.getRequestOrNull() ?: return RateLimitProperties() + val repoRateLimitAttribute = request.getAttribute(REPO_RATE_LIMIT_KEY) + if (repoRateLimitAttribute != null) { + require(repoRateLimitAttribute is RateLimitProperties) + return repoRateLimitAttribute + } + val repo = getRepoDetail() ?: return RateLimitProperties() + val receiveRateLimit = convertToDataSize(repo.configuration.getStringSetting(RECEIVE_RATE_LIMIT_OF_REPO)) + val responseRateLimit = convertToDataSize(repo.configuration.getStringSetting(RESPONSE_RATE_LIMIT_OF_REPO)) + val rateLimitProperties = RateLimitProperties(receiveRateLimit, responseRateLimit) + request.setAttribute(REPO_RATE_LIMIT_KEY, rateLimitProperties) + return rateLimitProperties + + } + + private fun convertToDataSize(dataSize: String?): DataSize { + if (dataSize.isNullOrEmpty()) return DataSize.ofBytes(-1) + return try { + DataSize.parse(dataSize) + } catch (e: Exception) { + DataSize.ofBytes(-1) + } + } } /** diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt index 2e0b879e6b..dc1c01d498 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt @@ -31,11 +31,13 @@ import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.exception.TooManyRequestsException import com.tencent.bkrepo.common.artifact.constant.X_CHECKSUM_MD5 import com.tencent.bkrepo.common.artifact.constant.X_CHECKSUM_SHA256 import com.tencent.bkrepo.common.artifact.exception.ArtifactResponseException import com.tencent.bkrepo.common.artifact.metrics.RecordAbleInputStream import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.stream.STREAM_BUFFER_SIZE @@ -50,6 +52,7 @@ import com.tencent.bkrepo.common.storage.monitor.Throughput import com.tencent.bkrepo.common.storage.monitor.measureThroughput import com.tencent.bkrepo.repository.pojo.node.NodeDetail import org.springframework.http.HttpMethod +import org.springframework.util.unit.DataSize import java.io.IOException import java.time.LocalDateTime import java.time.ZoneOffset @@ -68,6 +71,7 @@ open class DefaultArtifactResourceWriter( @Throws(ArtifactResponseException::class) override fun write(resource: ArtifactResource): Throughput { + responseRateLimitCheck() return if (resource.containsMultiArtifact()) { writeMultiArtifact(resource) } else { @@ -185,7 +189,7 @@ open class DefaultArtifactResourceWriter( val recordAbleInputStream = RecordAbleInputStream(inputStream) try { return measureThroughput { - recordAbleInputStream.rateLimit(storageProperties.response.rateLimit.toBytes()).use { + recordAbleInputStream.rateLimit(responseRateLimitWrapper(storageProperties.response.rateLimit)).use { it.copyTo( out = response.outputStream, bufferSize = getBufferSize(inputStream.range.length.toInt()) @@ -222,7 +226,9 @@ open class DefaultArtifactResourceWriter( resource.artifactMap.forEach { (name, inputStream) -> val recordAbleInputStream = RecordAbleInputStream(inputStream) zipOutput.putNextEntry(generateZipEntry(name, inputStream)) - recordAbleInputStream.rateLimit(storageProperties.response.rateLimit.toBytes()).use { + recordAbleInputStream.rateLimit( + responseRateLimitWrapper(storageProperties.response.rateLimit) + ).use { it.copyTo( out = zipOutput, bufferSize = getBufferSize(inputStream.range.length.toInt()) @@ -242,6 +248,31 @@ open class DefaultArtifactResourceWriter( } } + /** + * 将仓库级别的限速配置导入 + * 当同时存在全局限速配置以及仓库级别限速配置时,以仓库级别配置优先 + */ + private fun responseRateLimitWrapper(rateLimit: DataSize): Long { + val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo() + if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1)) { + return rateLimitOfRepo.responseRateLimit.toBytes() + } + return rateLimit.toBytes() + } + + /** + * 当仓库配置下载限速小于等于最低限速时则直接将请求断开, 避免占用过多连接 + */ + private fun responseRateLimitCheck() { + val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo() + if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1) && + rateLimitOfRepo.responseRateLimit <= storageProperties.response.circuitBreakerThreshold) { + throw TooManyRequestsException( + "The circuit breaker is activated when too many download requests are made to the service!" + ) + } + } + /** * 判断charset,一些媒体类型设置了charset会影响其表现,如application/vnd.android.package-archive * */ diff --git a/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt b/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt index 9801fb5902..1df6f9540c 100644 --- a/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt +++ b/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.common.service.exception import com.fasterxml.jackson.module.kotlin.MissingKotlinParameterException import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.exception.TooManyRequestsException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Response import org.springframework.core.Ordered @@ -137,6 +138,11 @@ class GlobalExceptionHandler : AbstractExceptionHandler() { return deferredResult } + @ExceptionHandler(TooManyRequestsException::class) + fun handleException(exception: TooManyRequestsException): Response { + return response(exception) + } + @ExceptionHandler(Exception::class) fun handleException(exception: Exception): Response { return response(exception) diff --git a/src/backend/common/common-service/src/main/resources/i18n/messages_en.properties b/src/backend/common/common-service/src/main/resources/i18n/messages_en.properties index 7fb1c1089b..c18e00958d 100644 --- a/src/backend/common/common-service/src/main/resources/i18n/messages_en.properties +++ b/src/backend/common/common-service/src/main/resources/i18n/messages_en.properties @@ -51,3 +51,4 @@ permission.project.denied = user[{0}] does not have [{1}] permission in project[ permission.repo.denied = user[{0}] does not have [{1}] permission in project[{2}] repo[{3}] operation.cross-cluster.not-allowed=Cross location operation is not allowed system.media-type.unacceptable=Unacceptable Media Type +too.many.requests=Too Many Requests: {0} \ No newline at end of file diff --git a/src/backend/common/common-service/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/common/common-service/src/main/resources/i18n/messages_zh_CN.properties index 4b561933d6..5a75fd4756 100644 --- a/src/backend/common/common-service/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/common/common-service/src/main/resources/i18n/messages_zh_CN.properties @@ -51,3 +51,4 @@ permission.project.denied=用户[{0}]没有[{1}]权限在[{2}]项目上 permission.repo.denied = 用户[{0}]没有[{1}]权限在[{2}]项目的[{3}]仓库上 operation.cross-cluster.not-allowed=不允许跨地点操作 system.media-type.unacceptable=不接受的Media Type +too.many.requests=请求过多: {0} \ No newline at end of file diff --git a/src/backend/common/common-service/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/common/common-service/src/main/resources/i18n/messages_zh_TW.properties index 6c5531d786..1c9e042d32 100644 --- a/src/backend/common/common-service/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/common/common-service/src/main/resources/i18n/messages_zh_TW.properties @@ -51,3 +51,4 @@ permission.project.denied=用戶[{0}]沒有[{1}]權限在[{2}]項目上 permission.repo.denied = 用戶[{0}]沒有[{1}]權限在[{2}]項目的[{3}]倉庫上 operation.cross-cluster.not-allowed=不允許跨地點操作 system.media-type.unacceptable=不接受的Media Type +too.many.requests=請求過多: {0} \ No newline at end of file diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/config/RateLimitProperties.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/config/RateLimitProperties.kt new file mode 100644 index 0000000000..6c747c7159 --- /dev/null +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/config/RateLimitProperties.kt @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.storage.core.config + +import org.springframework.util.unit.DataSize + +/** + * 上传、下载限速配置 + */ +data class RateLimitProperties( + /** + * 每秒接收数据量 + */ + var receiveRateLimit: DataSize = DataSize.ofBytes(-1), + /** + * 每秒传输数据量 + */ + var responseRateLimit: DataSize = DataSize.ofBytes(-1) +) diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/config/ResponseProperties.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/config/ResponseProperties.kt index fcfcf5e406..a55f8bee11 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/config/ResponseProperties.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/config/ResponseProperties.kt @@ -54,5 +54,9 @@ data class ResponseProperties( /** * 二进制媒体类型,不指定编码。 * */ - var binaryMediaTypes: Set = emptySet() + var binaryMediaTypes: Set = emptySet(), + /** + * 限速熔断阈值,当仓库配置的rateLimit小于等于限速熔断阈值时则直接将请求断开 + */ + var circuitBreakerThreshold: DataSize = DataSize.ofKilobytes(1), ) From ffb670f878bf1d5883ec1e24db6b9997ae8e7f89 Mon Sep 17 00:00:00 2001 From: v_wuqdwu Date: Tue, 29 Aug 2023 15:22:03 +0800 Subject: [PATCH 03/11] =?UTF-8?q?feat:=E4=BF=AE=E6=94=B9=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E7=94=A8=E6=88=B7id=E8=A7=84=E5=88=99=EF=BC=8C=E5=85=81?= =?UTF-8?q?=E8=AE=B8=E6=90=BA=E5=B8=A6@=E7=AC=A6=E5=8F=B7=20#1104?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/frontend/devops-repository/src/views/userManage/user.vue | 2 +- src/frontend/locale/repository/en-US.json | 2 +- src/frontend/locale/repository/zh-CN.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/frontend/devops-repository/src/views/userManage/user.vue b/src/frontend/devops-repository/src/views/userManage/user.vue index b10f4f7da6..a5520513d2 100644 --- a/src/frontend/devops-repository/src/views/userManage/user.vue +++ b/src/frontend/devops-repository/src/views/userManage/user.vue @@ -169,7 +169,7 @@ trigger: 'blur' }, { - regex: /^[a-zA-Z][a-zA-Z0-9_-]{1,31}$/, + regex: /^[a-zA-Z][a-zA-Z0-9_-|@]{1,31}$/, message: this.$t('account') + this.$t('space') + this.$t('include') + this.$t('space') + this.$t('userIdPlaceHolder'), trigger: 'blur' }, diff --git a/src/frontend/locale/repository/en-US.json b/src/frontend/locale/repository/en-US.json index dd34279d56..9a514b54af 100644 --- a/src/frontend/locale/repository/en-US.json +++ b/src/frontend/locale/repository/en-US.json @@ -162,7 +162,7 @@ "superAdmin": "Super admin", "modify": "Modify", "chineseName": "Cn name", - "userIdPlaceHolder": "English letters, numbers, _, -, starting with a letter", + "userIdPlaceHolder": "English letters, numbers, _, -,@, starting with a letter", "email": "Email", "loadingTips": "Loading", "cron": { diff --git a/src/frontend/locale/repository/zh-CN.json b/src/frontend/locale/repository/zh-CN.json index 087f0b723e..41e9932e66 100644 --- a/src/frontend/locale/repository/zh-CN.json +++ b/src/frontend/locale/repository/zh-CN.json @@ -162,7 +162,7 @@ "superAdmin": "管理员", "modify": "修改", "chineseName": "中文名", - "userIdPlaceHolder": "英文字母、数字、_、-,以字母开头", + "userIdPlaceHolder": "英文字母、数字、_、-,@,以字母开头", "email": "邮箱", "deleteUserTitle": "确认删除用户 {0} ?", "pwdPlacehodler": "长度8-32位,必须包含大小写字母、数字、特殊字符(除空格)", From c4b210ff16ff613986129ba59c73be5625013bbe Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 30 Aug 2023 15:33:52 +0800 Subject: [PATCH 04/11] =?UTF-8?q?bug:=20=E4=B8=B4=E6=97=B6=E6=96=B9?= =?UTF-8?q?=E6=A1=88=E9=81=BF=E5=85=8D=E4=BD=BF=E7=94=A8=E4=B8=B4=E6=97=B6?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E4=BB=8E=20generic=20=E4=B8=8B=E8=BD=BD=20do?= =?UTF-8?q?cker=20=E7=B1=BB=E5=9E=8B=E5=88=B6=E5=93=81=E6=8A=A5=E9=94=99#1?= =?UTF-8?q?092?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../artifact/repository/context/ArtifactContextHolder.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt index 6fa5a17a53..835a5f3c5e 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt @@ -295,7 +295,8 @@ class ArtifactContextHolder( require(repoRateLimitAttribute is RateLimitProperties) return repoRateLimitAttribute } - val repo = getRepoDetail() ?: return RateLimitProperties() + // 临时下载链接可能会使用 generic 下载其他业务类型的制品,此处先避免仓库找不到异常 + val repo = getRepoDetailOrNull() ?: return RateLimitProperties() val receiveRateLimit = convertToDataSize(repo.configuration.getStringSetting(RECEIVE_RATE_LIMIT_OF_REPO)) val responseRateLimit = convertToDataSize(repo.configuration.getStringSetting(RESPONSE_RATE_LIMIT_OF_REPO)) val rateLimitProperties = RateLimitProperties(receiveRateLimit, responseRateLimit) From 22d7ebba090516d292643edee132d5cbb83556af Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 30 Aug 2023 15:40:58 +0800 Subject: [PATCH 05/11] =?UTF-8?q?bug:=20=E6=B7=BB=E5=8A=A0=E6=B3=A8?= =?UTF-8?q?=E9=87=8A#1092?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/artifact/repository/context/ArtifactContextHolder.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt index 835a5f3c5e..b12afd7580 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt @@ -296,6 +296,7 @@ class ArtifactContextHolder( return repoRateLimitAttribute } // 临时下载链接可能会使用 generic 下载其他业务类型的制品,此处先避免仓库找不到异常 + // 此处修改潜在风险:当其他类型仓库使用 generic 服务的临时下载链接下载时无法控制住 val repo = getRepoDetailOrNull() ?: return RateLimitProperties() val receiveRateLimit = convertToDataSize(repo.configuration.getStringSetting(RECEIVE_RATE_LIMIT_OF_REPO)) val responseRateLimit = convertToDataSize(repo.configuration.getStringSetting(RESPONSE_RATE_LIMIT_OF_REPO)) From 1f5c2b4905dced6af4e47830e9fdcc02f0d38ee3 Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 30 Aug 2023 18:08:51 +0800 Subject: [PATCH 06/11] =?UTF-8?q?bug:=20=E4=BF=AE=E6=94=B9=20op=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E8=8A=82=E7=82=B9=E7=9B=B8=E5=85=B3=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E8=B6=85=E6=97=B6=E6=97=B6=E9=97=B4#1112?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/frontend/devops-op/src/api/node.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/frontend/devops-op/src/api/node.js b/src/frontend/devops-op/src/api/node.js index a5bdeba0eb..165179ed92 100644 --- a/src/frontend/devops-op/src/api/node.js +++ b/src/frontend/devops-op/src/api/node.js @@ -102,6 +102,7 @@ export function queryEmptyFolder(projectId, repoName, parentPath) { return request({ url: `${PREFIX_NODE_OPERATION}/emptyFolders/${projectId}/${repoName}`, method: 'get', + timeout: 600000, params: { parentFolder: parentPath } @@ -111,6 +112,7 @@ export function queryEmptyFolder(projectId, repoName, parentPath) { export function deleteEmptyFolder(projectId, repoName, parentPath) { return request({ url: `${PREFIX_NODE_OPERATION}/emptyFolders/${projectId}/${repoName}`, + timeout: 600000, method: 'delete', params: { parentFolder: parentPath @@ -121,6 +123,7 @@ export function deleteEmptyFolder(projectId, repoName, parentPath) { export function statisticalFirstLevelFolder(projectId, repoName, pageNumber) { return request({ url: `${PREFIX_NODE_OPERATION}/firstLevelFolder/${projectId}/${repoName}`, + timeout: 600000, method: 'get', params: { pageNumber: pageNumber, From ec3f7ba4eecd7d866fc4cb67e587bfdc2acc617e Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Thu, 31 Aug 2023 10:27:51 +0800 Subject: [PATCH 07/11] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E5=B7=B2?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0/=E4=B8=8B=E8=BD=BD=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F=E5=88=86=E5=B8=83=E6=8C=87=E6=A0=87=E7=9A=84?= =?UTF-8?q?=E7=BB=B4=E5=BA=A6=20#1060=20(#1097)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 优化已上传/下载文件大小分布指标的维度 #1060 * feat: 优化已上传/下载文件大小分布指标的维度 #1060 --- .../bkrepo/common/artifact/metrics/ArtifactMetrics.kt | 5 +++++ .../common/artifact/metrics/ArtifactMetricsProperties.kt | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactMetrics.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactMetrics.kt index 2f4fb1d1ee..37cbe876e4 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactMetrics.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactMetrics.kt @@ -59,9 +59,11 @@ class ArtifactMetrics( init { Companion.tagProvider = tagProvider Companion.meterRegistry = meterRegistry + Companion.properties = properties lruMeterFilter = LruMeterFilter(METER_LIMIT_PREFIX, meterRegistry, properties.maxMeters) meterRegistry.config().meterFilter(lruMeterFilter) } + override fun bindTo(meterRegistry: MeterRegistry) { Gauge.builder(ARTIFACT_UPLOADING_COUNT, uploadingCount) { it.get().toDouble() } .description(ARTIFACT_UPLOADING_COUNT_DESC) @@ -94,6 +96,7 @@ class ArtifactMetrics( private lateinit var tagProvider: ArtifactTransferTagProvider private lateinit var meterRegistry: MeterRegistry private lateinit var lruMeterFilter: LruMeterFilter + private lateinit var properties: ArtifactMetricsProperties private const val MAX_ATIME = 30.0 private const val BYTES = "bytes" private const val DAY = "day" @@ -107,6 +110,7 @@ class ArtifactMetrics( .description(ARTIFACT_UPLOADED_SIZE_DESC) .baseUnit(BYTES) .publishPercentileHistogram() + .maximumExpectedValue(properties.maxLe) .register(meterRegistry) } @@ -119,6 +123,7 @@ class ArtifactMetrics( .description(ARTIFACT_DOWNLOADED_SIZE_DESC) .baseUnit(BYTES) .publishPercentileHistogram() + .maximumExpectedValue(properties.maxLe) .register(meterRegistry) } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactMetricsProperties.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactMetricsProperties.kt index e69927b99f..039adf2019 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactMetricsProperties.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactMetricsProperties.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.common.artifact.metrics import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.util.unit.DataSize @ConfigurationProperties("management.metrics") data class ArtifactMetricsProperties( @@ -43,5 +44,9 @@ data class ArtifactMetricsProperties( /** * 是否通过日志清洗获取传输指标数据 */ - var collectByLog: Boolean = false + var collectByLog: Boolean = false, + /** + * 直方图le的最大值 + */ + var maxLe: Double = DataSize.ofMegabytes(100).toBytes().toDouble() ) From 6f74dd11ac262db05146916231fc166ad62fe91a Mon Sep 17 00:00:00 2001 From: gujunling <34835140+gujunling@users.noreply.github.com> Date: Thu, 31 Aug 2023 10:47:21 +0800 Subject: [PATCH 08/11] =?UTF-8?q?fix:=20=E5=88=B6=E5=93=81=E5=88=86?= =?UTF-8?q?=E5=8F=91=E5=9C=A8=E7=BC=96=E8=BE=91=E5=88=86=E5=8F=91=E8=AE=A1?= =?UTF-8?q?=E5=88=92=E6=97=B6=E7=9B=AE=E6=A0=87=E8=8A=82=E7=82=B9=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E5=B7=B2=E7=BB=8F=E8=A2=AB=E5=88=A0=E9=99=A4=E7=9A=84?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E5=AF=BC=E8=87=B4=E4=BF=9D=E5=AD=98=E6=97=B6?= =?UTF-8?q?=E6=8A=A5=E9=94=99=20#1088=20(#1089)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/views/planManage/createPlan/index.vue | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/frontend/devops-repository/src/views/planManage/createPlan/index.vue b/src/frontend/devops-repository/src/views/planManage/createPlan/index.vue index 811521c91f..7decf1060a 100644 --- a/src/frontend/devops-repository/src/views/planManage/createPlan/index.vue +++ b/src/frontend/devops-repository/src/views/planManage/createPlan/index.vue @@ -258,6 +258,8 @@ } }, created () { + // 需要获取最新的节点列表 + this.getClusterList() this.getRepoListAll({ projectId: this.projectId }) this.routeName !== 'createPlan' && this.handlePlanDetail() }, @@ -266,7 +268,8 @@ 'getRepoListAll', 'createPlan', 'getPlanDetail', - 'updatePlan' + 'updatePlan', + 'getClusterList' ]), formatDate, handlePlanDetail () { @@ -306,7 +309,7 @@ } : {}), conflictStrategy, - remoteClusterIds: remoteClusters.map(v => v.id), + remoteClusterIds: this.checkClusterExist(remoteClusters.map(v => v.id)), description, createdBy, createdDate @@ -316,6 +319,12 @@ this.isLoading = false }) }, + // 判断当前选择的节点是否存在于节点列表中 + checkClusterExist (arr) { + return arr.map(v => { + return this.clusterList.map(v => v.id).includes(v) && v + }).filter(Boolean) + }, changeReplicaObjectType () { this.replicaTaskObjects = [] this.planForm.executionStrategy === 'REAL_TIME' && (this.planForm.executionStrategy = 'IMMEDIATELY') @@ -325,6 +334,8 @@ this.$refs.planForm.clearError() }, async save () { + // 保存时也需要校验 + this.planForm.remoteClusterIds = this.checkClusterExist(this.planForm.remoteClusterIds) await this.$refs.planForm.validate() if (this.planForm.loading) return From 7f0affa478239c7fd72abdbe4813970c43bb2234 Mon Sep 17 00:00:00 2001 From: owen Date: Thu, 31 Aug 2023 11:21:59 +0800 Subject: [PATCH 09/11] =?UTF-8?q?feat:=20=E5=AE=B9=E5=99=A8=E5=8C=96?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E8=AE=BE=E7=BD=AEnfs=E6=8C=82=E8=BD=BD?= =?UTF-8?q?=E5=8F=82=E6=95=B0=20#1074=20(#1079)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 * feat: 容器化部署设置nfs挂载参数 #1074 --- .../bkrepo/templates/docker/deployment.yaml | 5 ++--- .../bkrepo/templates/fs-server/deployment.yaml | 5 ++--- .../bkrepo/templates/generic/deployment.yaml | 5 ++--- .../bkrepo/templates/helm/deployment.yaml | 5 ++--- .../bkrepo/templates/job/deployment.yaml | 5 ++--- .../bkrepo/templates/npm/deployment.yaml | 5 ++--- .../bkrepo/templates/opdata/deployment.yaml | 5 ++--- .../charts/bkrepo/templates/pv-nfs.yaml | 18 ++++++++++++++++++ .../charts/bkrepo/templates/pvc-nfs.yaml | 14 ++++++++++++++ .../bkrepo/templates/pypi/deployment.yaml | 5 ++--- .../templates/replication/deployment.yaml | 5 ++--- .../templates/repository/deployment.yaml | 5 ++--- .../kubernetes/charts/bkrepo/values.yaml | 1 + 13 files changed, 53 insertions(+), 30 deletions(-) create mode 100644 support-files/kubernetes/charts/bkrepo/templates/pv-nfs.yaml create mode 100644 support-files/kubernetes/charts/bkrepo/templates/pvc-nfs.yaml diff --git a/support-files/kubernetes/charts/bkrepo/templates/docker/deployment.yaml b/support-files/kubernetes/charts/bkrepo/templates/docker/deployment.yaml index 447fded405..649c639ec1 100644 --- a/support-files/kubernetes/charts/bkrepo/templates/docker/deployment.yaml +++ b/support-files/kubernetes/charts/bkrepo/templates/docker/deployment.yaml @@ -100,9 +100,8 @@ spec: volumes: - name: storage {{- if .Values.common.config.storage.nfs.enabled }} - nfs: - path: {{ .Values.common.config.storage.nfs.path }} - server: {{ .Values.common.config.storage.nfs.server }} + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-nfs-pvc {{- else if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "common.names.fullname" . }}-storage{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/templates/fs-server/deployment.yaml b/support-files/kubernetes/charts/bkrepo/templates/fs-server/deployment.yaml index 62a2ef68c5..e58b2b1747 100644 --- a/support-files/kubernetes/charts/bkrepo/templates/fs-server/deployment.yaml +++ b/support-files/kubernetes/charts/bkrepo/templates/fs-server/deployment.yaml @@ -100,9 +100,8 @@ spec: volumes: - name: storage {{- if .Values.common.config.storage.nfs.enabled }} - nfs: - path: {{ .Values.common.config.storage.nfs.path }} - server: {{ .Values.common.config.storage.nfs.server }} + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-nfs-pvc {{- else if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "common.names.fullname" . }}-storage{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/templates/generic/deployment.yaml b/support-files/kubernetes/charts/bkrepo/templates/generic/deployment.yaml index 2c9f9f7890..0519696fd4 100644 --- a/support-files/kubernetes/charts/bkrepo/templates/generic/deployment.yaml +++ b/support-files/kubernetes/charts/bkrepo/templates/generic/deployment.yaml @@ -124,9 +124,8 @@ spec: volumes: - name: storage {{- if .Values.common.config.storage.nfs.enabled }} - nfs: - path: {{ .Values.common.config.storage.nfs.path }} - server: {{ .Values.common.config.storage.nfs.server }} + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-nfs-pvc {{- else if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "common.names.fullname" . }}-storage{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/templates/helm/deployment.yaml b/support-files/kubernetes/charts/bkrepo/templates/helm/deployment.yaml index b35d5c5664..03b52f7a7d 100644 --- a/support-files/kubernetes/charts/bkrepo/templates/helm/deployment.yaml +++ b/support-files/kubernetes/charts/bkrepo/templates/helm/deployment.yaml @@ -100,9 +100,8 @@ spec: volumes: - name: storage {{- if .Values.common.config.storage.nfs.enabled }} - nfs: - path: {{ .Values.common.config.storage.nfs.path }} - server: {{ .Values.common.config.storage.nfs.server }} + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-nfs-pvc {{- else if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "common.names.fullname" . }}-storage{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/templates/job/deployment.yaml b/support-files/kubernetes/charts/bkrepo/templates/job/deployment.yaml index b7b00df296..5911d61bd6 100644 --- a/support-files/kubernetes/charts/bkrepo/templates/job/deployment.yaml +++ b/support-files/kubernetes/charts/bkrepo/templates/job/deployment.yaml @@ -100,9 +100,8 @@ spec: volumes: - name: storage {{- if .Values.common.config.storage.nfs.enabled }} - nfs: - path: {{ .Values.common.config.storage.nfs.path }} - server: {{ .Values.common.config.storage.nfs.server }} + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-nfs-pvc {{- else if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "common.names.fullname" . }}-storage{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/templates/npm/deployment.yaml b/support-files/kubernetes/charts/bkrepo/templates/npm/deployment.yaml index 0cace081e8..812754adb8 100644 --- a/support-files/kubernetes/charts/bkrepo/templates/npm/deployment.yaml +++ b/support-files/kubernetes/charts/bkrepo/templates/npm/deployment.yaml @@ -100,9 +100,8 @@ spec: volumes: - name: storage {{- if .Values.common.config.storage.nfs.enabled }} - nfs: - path: {{ .Values.common.config.storage.nfs.path }} - server: {{ .Values.common.config.storage.nfs.server }} + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-nfs-pvc {{- else if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "common.names.fullname" . }}-storage{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/templates/opdata/deployment.yaml b/support-files/kubernetes/charts/bkrepo/templates/opdata/deployment.yaml index 39bd028e23..ad86e925b1 100644 --- a/support-files/kubernetes/charts/bkrepo/templates/opdata/deployment.yaml +++ b/support-files/kubernetes/charts/bkrepo/templates/opdata/deployment.yaml @@ -100,9 +100,8 @@ spec: volumes: - name: storage {{- if .Values.common.config.storage.nfs.enabled }} - nfs: - path: {{ .Values.common.config.storage.nfs.path }} - server: {{ .Values.common.config.storage.nfs.server }} + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-nfs-pvc {{- else if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "common.names.fullname" . }}-storage{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/templates/pv-nfs.yaml b/support-files/kubernetes/charts/bkrepo/templates/pv-nfs.yaml new file mode 100644 index 0000000000..0bce32af01 --- /dev/null +++ b/support-files/kubernetes/charts/bkrepo/templates/pv-nfs.yaml @@ -0,0 +1,18 @@ +{{- if .Values.common.config.storage.nfs.enabled -}} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ include "common.names.fullname" . }}-nfs-pv +spec: + storageClassName: bkrepo-nfs + capacity: + storage: {{ .Values.common.config.storage.nfs.capacity }} + accessModes: + - ReadWriteMany + mountOptions: + - lookupcache=pos + nfs: + server: {{ .Values.common.config.storage.nfs.server }} + path: {{ .Values.common.config.storage.nfs.path }} +{{- end -}} + diff --git a/support-files/kubernetes/charts/bkrepo/templates/pvc-nfs.yaml b/support-files/kubernetes/charts/bkrepo/templates/pvc-nfs.yaml new file mode 100644 index 0000000000..49fdc1b369 --- /dev/null +++ b/support-files/kubernetes/charts/bkrepo/templates/pvc-nfs.yaml @@ -0,0 +1,14 @@ +{{- if .Values.common.config.storage.nfs.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "common.names.fullname" . }}-nfs-pvc +spec: + storageClassName: bkrepo-nfs + resources: + requests: + storage: {{ .Values.common.config.storage.nfs.capacity }} + accessModes: + - ReadWriteMany + volumeName: {{ include "common.names.fullname" . }}-nfs-pv +{{- end -}} \ No newline at end of file diff --git a/support-files/kubernetes/charts/bkrepo/templates/pypi/deployment.yaml b/support-files/kubernetes/charts/bkrepo/templates/pypi/deployment.yaml index 8a73a9f401..ed298c41e1 100644 --- a/support-files/kubernetes/charts/bkrepo/templates/pypi/deployment.yaml +++ b/support-files/kubernetes/charts/bkrepo/templates/pypi/deployment.yaml @@ -100,9 +100,8 @@ spec: volumes: - name: storage {{- if .Values.common.config.storage.nfs.enabled }} - nfs: - path: {{ .Values.common.config.storage.nfs.path }} - server: {{ .Values.common.config.storage.nfs.server }} + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-nfs-pvc {{- else if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "common.names.fullname" . }}-storage{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/templates/replication/deployment.yaml b/support-files/kubernetes/charts/bkrepo/templates/replication/deployment.yaml index 9054a98406..8d3b21bb0b 100644 --- a/support-files/kubernetes/charts/bkrepo/templates/replication/deployment.yaml +++ b/support-files/kubernetes/charts/bkrepo/templates/replication/deployment.yaml @@ -103,9 +103,8 @@ spec: volumes: - name: storage {{- if .Values.common.config.storage.nfs.enabled }} - nfs: - path: {{ .Values.common.config.storage.nfs.path }} - server: {{ .Values.common.config.storage.nfs.server }} + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-nfs-pvc {{- else if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "common.names.fullname" . }}-storage{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/templates/repository/deployment.yaml b/support-files/kubernetes/charts/bkrepo/templates/repository/deployment.yaml index a7be1cc1ef..ab6d6160c8 100644 --- a/support-files/kubernetes/charts/bkrepo/templates/repository/deployment.yaml +++ b/support-files/kubernetes/charts/bkrepo/templates/repository/deployment.yaml @@ -115,9 +115,8 @@ spec: volumes: - name: storage {{- if .Values.common.config.storage.nfs.enabled }} - nfs: - path: {{ .Values.common.config.storage.nfs.path }} - server: {{ .Values.common.config.storage.nfs.server }} + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-nfs-pvc {{- else if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "common.names.fullname" . }}-storage{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/values.yaml b/support-files/kubernetes/charts/bkrepo/values.yaml index f746e4aecc..86bf1d9e50 100644 --- a/support-files/kubernetes/charts/bkrepo/values.yaml +++ b/support-files/kubernetes/charts/bkrepo/values.yaml @@ -212,6 +212,7 @@ common: # 是否启用nfs挂载 nfs: enabled: false + capacity: 100Ti path: / server: From a1e0e00f1be105e5cb1cf5a7a62e0c080245e742 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Thu, 31 Aug 2023 11:48:22 +0800 Subject: [PATCH 10/11] =?UTF-8?q?feat:=20=E9=95=9C=E5=83=8F=E5=88=B6?= =?UTF-8?q?=E5=93=81=E5=88=A0=E9=99=A4=E4=BC=98=E5=8C=96=20#203=20(#582)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bug: 添加第三方镜像仓库package拉取扩展点 #462 * bug: 镜像仓库支持remote代理 #462 * bug: 接口调整 #462 * bug: 引用调整 #462 * bug: 代理url校验 #462 * bug: 代码优化 #462 * bug: 引用调整 #462 * bug: 代码调整 #462 * bug: 代码调整 #462 * bug: token缓存key调整 #462 * bug: 代码调整 #462 * bug: 代码调整 #462 * feat: 代码调整 #203 * feat: blob存储节点路径调整 #203 * feat: blob刷新接口 #203 * feat: 包与版本删除逻辑调整 #203 * feat: 方法名调整 #203 * feat: 版本删除路径调整 #203 * feat: 接口调整 #203 * feat: 删除逻辑调整 #203 * feat: blob节点刷新代码调整 #203 * feat: 增加打印 #203 * feat: 获取节点路径调整 #203 * feat: 节点刷新接口逻辑调整 #203 * feat: 代码调整 #203 * feat: 代码调整 #203 * feat: 去掉注释 #203 * feat: 代码调整 #203 * feat: 禁止v1的镜像manifest文件上传,部分客户端拉取会报错 #908 * bug: feign请求字段返回先转换成String再进行类型转换 #892 (cherry picked from commit ec4614784cb5bb0b8b2149dfec094cd53df93c37) * bug: catalog相关接口调整 #203 * bug: 路径拼接调整 #203 * bug: 当包下没有版本时需要删掉包目录,清除存储在blobs目录下数据 #203 * feat: 历史数据修正方案调整 #203 * bug: 解决代码冲突#203 * bug: 代码扫描问题修复#203 * bug: 代码问题修复#203 * bug: 代码检查问题修复#203 * bug: mount上传增加元数据,用于区别新老 blob 存储路径#203 * bug: 镜像刷新前先判断是否已经是新版本路径#203 * feat: 镜像blob路径调整后,分发对应获取路径调整 #203 --- .../common/api/constant/AuthenticationKeys.kt | 34 + .../authentication}/AuthenticationProperty.kt | 2 +- .../common/api/util/AuthenticationUtil.kt | 74 ++ .../repository/remote/RemoteRepository.kt | 1 + .../common/artifact/util/http/UrlFormatter.kt | 89 ++ .../artifact/util/http/UrlFormatterTest.kt | 251 ++++++ .../bkrepo/job/batch/OciBlobNodeRefreshJob.kt | 139 +++ .../tencent/bkrepo/job/batch/RepoInitJob.kt | 26 +- .../bkrepo/job/batch/RepoRefreshJob.kt | 26 +- .../bkrepo/job/batch/base/DefaultRepoJob.kt | 1 + .../OciBlobNodeRefreshJobProperties.kt | 40 + .../config/properties/RepoJobProperties.kt | 2 +- .../com/tencent/bkrepo/oci/api/OciClient.kt | 28 +- .../bkrepo/oci/constant/OciConstants.kt | 14 +- .../bkrepo/oci/constant/OciMessageCode.kt | 5 +- .../ImagePackageInfoPullExtension.kt | 43 + .../oci/extension/ImagePackagePullContext.kt | 38 + .../oci/pojo/artifact/OciArtifactInfo.kt | 8 +- .../pojo/metadata/HelmMaintainerMetadata.kt | 43 - .../oci/pojo/remote/RemoteRequestProperty.kt | 39 + .../oci/pojo/response/ResponseProperty.kt | 40 + .../bkrepo/oci/util/OciLocationUtils.kt | 38 +- .../com/tencent/bkrepo/oci/util/OciUtils.kt | 14 - .../repository/OciRegistryLocalRepository.kt | 118 ++- .../repository/OciRegistryRemoteRepository.kt | 334 ++++--- .../resolver/OciTagArtifactInfoResolver.kt | 13 +- .../service/OciPackageController.kt | 36 +- .../oci/controller/user/CatalogController.kt | 22 +- .../oci/controller/user/OciBlobController.kt | 2 +- .../oci/controller/user/OciTagController.kt | 10 +- .../oci/controller/user/UserOciController.kt | 2 +- .../bkrepo/oci/listener/base/EventExecutor.kt | 17 +- .../consumer/RemoteImageRepoEventConsumer.kt} | 69 +- .../consumer/ReplicationEventListener.kt | 11 +- .../ThirdPartyReplicationEventConsumer.kt | 13 +- .../bkrepo/oci/service/OciOperationService.kt | 74 +- .../oci/service/impl/OciBlobServiceImpl.kt | 70 +- .../service/impl/OciOperationServiceImpl.kt | 819 +++++++++--------- .../bkrepo/oci/util/ObjectBuildUtils.kt | 13 +- .../bkrepo/oci/util/OciResponseUtils.kt | 93 +- .../resources/i18n/messages_en.properties | 5 +- .../resources/i18n/messages_zh_CN.properties | 5 +- .../resources/i18n/messages_zh_TW.properties | 5 +- .../bkrepo/replication/constant/Constants.kt | 5 +- .../handler/ArtifactReplicationHandler.kt | 6 +- ...RemoteClusterArtifactReplicationHandler.kt | 4 +- .../internal/type/DockerPackageNodeMapper.kt | 40 +- .../type/helm/HelmArtifactPushClient.kt | 4 +- .../type/oci/OciAuthorizationService.kt | 53 +- .../service/impl/RemoteNodeServiceImpl.kt | 2 +- .../bkrepo/replication/util/HttpUtils.kt | 64 +- .../bkrepo/repository/api/NodeClient.kt | 12 +- .../controller/service/NodeController.kt | 6 + .../service/node/NodeRestoreOperation.kt | 4 + .../service/node/impl/NodeRestoreSupport.kt | 7 + .../service/node/impl/NodeServiceImpl.kt | 6 + .../node/impl/edge/EdgeNodeServiceImpl.kt | 4 + .../bkrepo/repository/util/NodeQueryHelper.kt | 11 + 58 files changed, 1923 insertions(+), 1031 deletions(-) create mode 100644 src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt rename src/backend/{replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote => common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication}/AuthenticationProperty.kt (96%) create mode 100644 src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt create mode 100644 src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt delete mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/response/ResponseProperty.kt rename src/backend/oci/{api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmChartMetadata.kt => biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt} (53%) diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt new file mode 100644 index 0000000000..070361adef --- /dev/null +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.api.constant + +object AuthenticationKeys { + const val BEARER_REALM = "Bearer realm" + const val SERVICE = "service" + const val SCOPE = "scope" +} \ No newline at end of file diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/AuthenticationProperty.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication/AuthenticationProperty.kt similarity index 96% rename from src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/AuthenticationProperty.kt rename to src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication/AuthenticationProperty.kt index 47e4268a7f..5d10b119fe 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/AuthenticationProperty.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication/AuthenticationProperty.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.pojo.remote +package com.tencent.bkrepo.common.api.pojo.authentication /** * 返回头中的WWW_AUTHENTICATE字段包含的属性 diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt new file mode 100644 index 0000000000..d97bdb2e44 --- /dev/null +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.api.util + +import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.BEARER_REALM +import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.SCOPE +import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.SERVICE +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.pojo.authentication.AuthenticationProperty + +object AuthenticationUtil { + + /** + * 解析返回头中的WWW_AUTHENTICATE字段, 只针对为Bearer realm + */ + fun parseWWWAuthenticateHeader(wwwAuthenticate: String, scope: String?): AuthenticationProperty? { + val map: MutableMap = mutableMapOf() + return try { + val params = wwwAuthenticate.split("\",") + params.forEach { + val param = it.split(Regex("="),2) + val name = param.first() + val value = param.last().replace("\"", "") + map[name] = value + } + AuthenticationProperty( + authUrl = map[BEARER_REALM]!!, + service = map[SERVICE]!!, + scope = scope + ) + } catch (e: Exception) { + null + } + } + + fun buildAuthenticationUrl(property: AuthenticationProperty, userName: String?): String? { + if (property.authUrl.isBlank()) return null + var result = if (property.authUrl.contains(StringPool.QUESTION)) { + "${property.authUrl}&$SERVICE=${property.service}" + } else { + "${property.authUrl}?$SERVICE=${property.service}" + } + property.scope?.let { + result += "&$SCOPE=${property.scope}" + } + userName?.let { result += "&account=$userName" } + return result + } +} \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt index 5baf2589b5..7f9e3ffd5f 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt @@ -233,6 +233,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() { if (addInterceptor) { createAuthenticateInterceptor(configuration.credentials)?.let { builder.addInterceptor(it) } } + builder.retryOnConnectionFailure(true) return builder.build() } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt index da5a5b6217..6b540cda8c 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt @@ -31,10 +31,16 @@ package com.tencent.bkrepo.common.artifact.util.http +import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.api.constant.CharPool.QUESTION import com.tencent.bkrepo.common.api.constant.CharPool.SLASH +import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.constant.StringPool.HTTP import com.tencent.bkrepo.common.api.constant.StringPool.HTTPS +import com.tencent.bkrepo.common.storage.innercos.retry +import java.net.HttpURLConnection +import java.net.MalformedURLException +import java.net.URL /** * Http URL 格式化工具类 @@ -82,4 +88,87 @@ object UrlFormatter { } return url } + + + /** + * 拼接url + */ + fun buildUrl( + url: String, + path: String = StringPool.EMPTY, + params: String = StringPool.EMPTY, + ): String { + if (url.isBlank()) + throw IllegalArgumentException("Url should not be blank") + val newUrl = addProtocol(url.trim().trimEnd(SLASH)) + val baseUrl = URL(newUrl, newUrl.path) + val builder = StringBuilder(baseUrl.toString().trimEnd(SLASH)) + if (path.isNotBlank()) { + builder.append(SLASH).append(path.trimStart(SLASH)) + } + if (!newUrl.query.isNullOrEmpty()) { + builder.append(QUESTION).append(newUrl.query) + } + return addParams(builder.toString(), params) + } + + fun addParams(url: String, params: String): String { + val baseUrl = URL(url) + val builder = StringBuilder(baseUrl.toString()) + + if (params.isNotEmpty()) { + if (builder.contains(QUESTION)) { + builder.append(CharPool.AND).append(params) + } else { + builder.append(QUESTION).append(params) + } + } + return builder.toString() + } + + /** + * 当没有protocol时进行添加 + */ + fun addProtocol(registry: String): URL { + try { + return URL(registry) + } catch (ignore: MalformedURLException) { + } + return addProtocolToHost(registry) + } + + /** + * 针对url如果没传protocol, 则默认以https请求发送; + * 如果http请求无法访问,则以http发送 + */ + private fun addProtocolToHost(registry: String): URL { + val url = try { + URL("$HTTPS$registry") + } catch (ignore: MalformedURLException) { + throw IllegalArgumentException("Check your input url!") + } + return try { + retry(times = 3, delayInSeconds = 1) { + validateHttpsProtocol(url) + url + } + } catch (ignore: Exception) { + URL(url.toString().replaceFirst("^https".toRegex(), "http")) + } + } + + /** + * 验证registry是否支持https + */ + private fun validateHttpsProtocol(url: URL): Boolean { + return try { + val http: HttpURLConnection = url.openConnection() as HttpURLConnection + http.instanceFollowRedirects = false + http.responseCode + http.disconnect() + true + } catch (e: Exception) { + throw e + } + } } diff --git a/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt new file mode 100644 index 0000000000..1ddb218236 --- /dev/null +++ b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt @@ -0,0 +1,251 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.artifact.util.http + +import com.tencent.bkrepo.common.api.constant.StringPool +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertAll + +internal class UrlFormatterTest { + + @Test + fun buildUrl() { + val url = "" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + try { + val result = UrlFormatter.buildUrl(url, path, params) + } catch (e: Exception) { + assertTrue(e.message!!.contains("Url should not be blank")) + } + + } + + @Test + fun buildUrl1() { + val url = "bkrepo.example.com" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com", result)} + ) + } + + @Test + fun buildUrl2() { + val url = "http://bkrepo.example.com" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com", result)} + ) + } + + @Test + fun buildUrl3() { + val url = "http://bkrepo.example.com/" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com", result)} + ) + } + + @Test + fun buildUrl4() { + val url = "http://bkrepo.example.com//" + val path = "/v2/" + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2/", result)} + ) + } + + @Test + fun buildUrl5() { + val url = "http://bkrepo.example.com//" + val path = "v2" + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2", result)} + ) + } + + @Test + fun buildUrl6() { + val url = "http://bkrepo.example.com//" + val path = "v2" + val params = "a=a" + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2?a=a", result)} + ) + } + + @Test + fun buildUrl7() { + val url = "http://bkrepo.example.com/?b=b" + val path = "v2" + val params = "a=a" + + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2?b=b&a=a", result)} + ) + } + + @Test + fun buildUrl8() { + val url = "http://bkrepo.example.com/test/" + val path = "/v2" + + val result = UrlFormatter.buildUrl(url, path) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2", result)} + ) + } + + @Test + fun buildUrl9() { + val url = "http://bkrepo.example.com/test/" + val path = "/v2/" + + val result = UrlFormatter.buildUrl(url, path) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2/", result)} + ) + } + + @Test + fun buildUrl10() { + val url = "http://bkrepo.example.com/test/" + val path = "/v2/" + val params = "a=a" + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2/?a=a", result)} + ) + } + + @Test + fun buildUrl11() { + val url = "http://bkrepo.example.com/test/?b=b" + val path = "/v2/" + val params = "a=a" + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2/?b=b&a=a", result)} + ) + } + + @Test + fun addParams() { + val url = "http://bkrepo.example.com/test/?b=b" + val params = "a=a" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/?b=b&a=a", result)} + ) + } + + @Test + fun addParams1() { + val url = "http://bkrepo.example.com/test/?b=b" + val params = "" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/?b=b", result)} + ) + } + + @Test + fun addParams2() { + val url = "http://bkrepo.example.com/test/" + val params = "b=b" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/?b=b", result)} + ) + } + + @Test + fun addParams3() { + val url = "http://bkrepo.example.com/test" + val params = "b=b" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test?b=b", result)} + ) + } + + + + @Test + fun addProtocol() { + val url = "bkrepo.example.com/test/" + val result = UrlFormatter.addProtocol(url).toString() + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/", result)} + ) + } + + @Test + fun addProtocol1() { + val url = "http://bkrepo.example.com/test/" + val result = UrlFormatter.addProtocol(url).toString() + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/", result)} + ) + } + + @Test + fun addProtocol2() { + val url = "https://bkrepo.example.com/test/" + val result = UrlFormatter.addProtocol(url).toString() + assertAll( + {Assertions.assertEquals("https://bkrepo.example.com/test/", result)} + ) + } + + @Test + fun addProtocol3() { + val url = "bkrepo.example.com:test/" + try { + val result = UrlFormatter.addProtocol(url).toString() + } catch (e: Exception) { + assertTrue(e.message!!.contains("Check your input url!")) + } + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt new file mode 100644 index 0000000000..68a139804a --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt @@ -0,0 +1,139 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch + +import com.tencent.bkrepo.common.service.log.LoggerHolder +import com.tencent.bkrepo.job.TYPE +import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob +import com.tencent.bkrepo.job.batch.base.JobContext +import com.tencent.bkrepo.job.config.properties.OciBlobNodeRefreshJobProperties +import com.tencent.bkrepo.job.exception.JobExecuteException +import com.tencent.bkrepo.oci.api.OciClient +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.find +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +/** + * 用于将存储在blobs目录下的公共blob节点全部迁移到对应版本目录下, + * 即从/{packageName}/blobs/xxx 到/{packageName}/blobs/{tag}/xxx + */ +@Component +@EnableConfigurationProperties(OciBlobNodeRefreshJobProperties::class) +class OciBlobNodeRefreshJob( + private val properties: OciBlobNodeRefreshJobProperties, + private val mongoTemplate: MongoTemplate, + private val ociClient: OciClient +) : DefaultContextMongoDbJob(properties) { + private val types: List + get() = properties.repositoryTypes + + override fun start(): Boolean { + return super.start() + } + + override fun entityClass(): Class { + return PackageData::class.java + } + + override fun collectionNames(): List { + return listOf(COLLECTION_NAME) + } + + override fun buildQuery(): Query { + return Query( + Criteria.where(TYPE).`in`(properties.repositoryTypes) + ) + } + + override fun run(row: PackageData, collectionName: String, context: JobContext) { + with(row) { + try { + val result = mongoTemplate.find>( + Query(Criteria(PACKAGE_ID).isEqualTo(row.id)), + PACKAGE_VERSION_NAME + ) + if (result.isEmpty()) return + var refreshStatus = true + for (map in result) { + val version = map[NAME] as String? ?: continue + logger.info( + "Preparing to send blob refresh request for package ${row.name}|${version}" + + " in repo ${row.projectId}|${row.repoName}." + ) + refreshStatus = refreshStatus && ociClient.blobPathRefresh( + projectId = row.projectId, + repoName = row.repoName, + packageName = row.name, + version = version + ).data ?: false + } + if (refreshStatus) { + // 当包下版本对应镜像的 blob节点都刷新完成后,删除旧路径下的 blob文件 + logger.info( + "Will delete blobs folder of package ${row.name}" + + " in repo ${row.projectId}|${row.repoName}." + ) + ociClient.deleteBlobsFolderAfterRefreshed(projectId, repoName, row.name) + } + } catch (e: Exception) { + throw JobExecuteException( + "Failed to send blob refresh request for package ${row.name}" + + " in repo ${row.projectId}|${row.repoName}, error: ${e.message}", e + ) + } + } + } + + data class PackageData(private val map: Map) { + val id: String by map + val repoName: String by map + val projectId: String by map + val name: String by map + val key: String by map + val type: String by map + } + + companion object { + private val logger = LoggerHolder.jobLogger + const val COLLECTION_NAME = "package" + private const val PACKAGE_VERSION_NAME = "package_version" + private const val METADATA_KEY = "blobPathRefreshed" + private const val METADATA = "metadata" + private const val PACKAGE_ID = "packageId" + private const val NAME = "name" + + } + + override fun mapToEntity(row: Map): PackageData { + return PackageData(row) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt index e4ba11d578..d796260574 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.batch import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration import com.tencent.bkrepo.common.service.log.LoggerHolder import com.tencent.bkrepo.helm.api.HelmClient @@ -38,6 +39,7 @@ import com.tencent.bkrepo.job.batch.base.DefaultRepoJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.config.properties.RepoInitJobProperties import com.tencent.bkrepo.job.exception.JobExecuteException +import com.tencent.bkrepo.oci.api.OciClient import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query @@ -51,7 +53,8 @@ import java.time.LocalDateTime @EnableConfigurationProperties(RepoInitJobProperties::class) class RepoInitJob( private val properties: RepoInitJobProperties, - private val helmClient: HelmClient + private val helmClient: HelmClient, + private val ociClient: OciClient ) : DefaultRepoJob(properties) { private val types: List @@ -74,16 +77,21 @@ class RepoInitJob( } override fun run(row: ProxyRepoData, collectionName: String, context: JobContext) { - with(row) { - try { - val config = configuration.readJsonString() - if (checkConfigType(config)) { - logger.info("Init request will be sent in repo $projectId|$name") - helmClient.initIndexAndPackage(projectId, name) + try { + val config = row.configuration.readJsonString() + if (!checkConfigType(config)) return + logger.info("init request will be sent in repo ${row.projectId}|${row.name}") + when (row.type) { + RepositoryType.HELM.name -> { + helmClient.initIndexAndPackage(row.projectId, row.name) } - } catch (e: Exception) { - throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) + RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { + ociClient.getPackagesFromThirdPartyRepo(row.projectId, row.name) + } + else -> throw UnsupportedOperationException() } + } catch (e: Exception) { + throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt index 23a1776c85..b786b23a8a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.batch import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration import com.tencent.bkrepo.common.service.log.LoggerHolder import com.tencent.bkrepo.helm.api.HelmClient @@ -37,6 +38,7 @@ import com.tencent.bkrepo.job.batch.base.DefaultRepoJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.config.properties.RepoRefreshJobProperties import com.tencent.bkrepo.job.exception.JobExecuteException +import com.tencent.bkrepo.oci.api.OciClient import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query @@ -49,7 +51,8 @@ import org.springframework.stereotype.Component @EnableConfigurationProperties(RepoRefreshJobProperties::class) class RepoRefreshJob( private val properties: RepoRefreshJobProperties, - private val helmClient: HelmClient + private val helmClient: HelmClient, + private val ociClient: OciClient ) : DefaultRepoJob(properties) { private val types: List @@ -70,16 +73,21 @@ class RepoRefreshJob( } override fun run(row: ProxyRepoData, collectionName: String, context: JobContext) { - with(row) { - try { - val config = configuration.readJsonString() - if (checkConfigType(config)) { - logger.info("Refresh request will be sent in repo $projectId|$name") - helmClient.refreshIndexYamlAndPackage(projectId, name) + try { + val config = row.configuration.readJsonString() + if (!checkConfigType(config)) return + logger.info("Refresh request will be sent in repo ${row.projectId}|${row.name}") + when (row.type) { + RepositoryType.HELM.name -> { + helmClient.refreshIndexYamlAndPackage(row.projectId, row.name) } - } catch (e: Exception) { - throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) + RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { + ociClient.getPackagesFromThirdPartyRepo(row.projectId, row.name) + } + else -> throw UnsupportedOperationException() } + } catch (e: Exception) { + throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt index 77fa5df3a4..13c006d39c 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt @@ -66,6 +66,7 @@ abstract class DefaultRepoJob( data class ProxyRepoData(private val map: Map) { val name: String by map val projectId: String by map + val type: String by map val configuration: String by map } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt new file mode 100644 index 0000000000..f22e3f3f96 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.config.properties + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties("job.oci-blob-node-refresh") +class OciBlobNodeRefreshJobProperties( + /** + * 需要进行远端分发集群推送的仓库类型 + * */ + var repositoryTypes: List = listOf("OCI", "DOCKER"), + override var enabled: Boolean = true, + override var cron: String = "0 0 4/24 * * ?" + ) : MongodbJobProperties() diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt index 778f07edca..386f585b21 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt @@ -36,5 +36,5 @@ open class RepoJobProperties( /** * 需要特殊处理的仓库类型 * */ - var repositorytypes: List = listOf("HELM") + var repositorytypes: List = listOf("HELM", "OCI", "DOCKER") ) : MongodbJobProperties() diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt index 380d3027dd..b911c0df8a 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt @@ -34,10 +34,12 @@ import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import org.springframework.cloud.openfeign.FeignClient import org.springframework.context.annotation.Primary +import org.springframework.web.bind.annotation.DeleteMapping +import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestMapping - +import org.springframework.web.bind.annotation.RequestParam @Api("oci") @@ -51,4 +53,28 @@ interface OciClient { fun packageCreate( @RequestBody record: OciReplicationRecordInfo ): Response + + @ApiOperation("定时从第三方仓库拉取对应的package信息") + @PostMapping("/pull/package/{projectId}/{repoName}") + fun getPackagesFromThirdPartyRepo( + @PathVariable projectId: String, + @PathVariable repoName: String + ): Response + + @ApiOperation("刷新对应版本镜像的blob节点路径") + @PostMapping("/blob/path/refresh/{projectId}/{repoName}") + fun blobPathRefresh( + @PathVariable projectId: String, + @PathVariable repoName: String, + @RequestParam packageName: String, + @RequestParam version: String, + ): Response + + @ApiOperation("当历史数据刷新完成后,删除blobs路径下的公共blob节点") + @DeleteMapping("/blobs/delete/{projectId}/{repoName}") + fun deleteBlobsFolderAfterRefreshed( + @PathVariable projectId: String, + @PathVariable repoName: String, + @RequestParam packageName: String + ): Response } diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt index 9b2a5201dc..c4e56a21fe 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt @@ -42,9 +42,6 @@ const val HTTP_FORWARDED_PROTO = "X-Forwarded-Proto" const val HTTP_PROTOCOL_HTTP = "http" const val HTTP_PROTOCOL_HTTPS = "https" const val HOST = "Host" -const val BEARER_REALM = "Bearer realm" -const val SERVICE = "service" -const val SCOPE = "scope" const val PATCH = "PATCH" const val POST = "POST" const val NODE_FULL_PATH = "fullPath" @@ -53,6 +50,10 @@ const val N = "n" const val DOCKER_LINK = "Link" const val PROXY_URL = "proxyUrl" +const val BLOB_PATH_VERSION_KEY = "blobPathVersion" +const val BLOB_PATH_VERSION_VALUE = "v1" + +const val BLOB_PATH_REFRESHED_KEY = "blobPathRefreshed" const val MANIFEST = "manifest.json" const val MEDIA_TYPE = "mediaType" @@ -94,6 +95,7 @@ const val LAST_MODIFIED_BY = "lastModifiedBy" const val LAST_MODIFIED_DATE = "lastModifiedDate" const val DOWNLOADS = "downloads" const val MD5 = "md5" +const val DELETED = "deleted" const val EMPTY_FILE_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" @@ -101,6 +103,10 @@ const val EMPTY_FILE_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca const val OCI_MANIFEST = "manifest.json" const val STAGE_TAG = "stageTag" +const val TAG_LIST_REQUEST = "tagList" +const val CATALOG_REQUEST = "catalog" +const val REQUEST_IMAGE = "image" + // OCIScheme is the URL scheme for OCI-based requests const val OCI_SCHEME = "oci" @@ -121,6 +127,8 @@ const val LEGACY_CHART_LAYER_MEDIA_TYPE = "application/tar+gzip" const val OCI_IMAGE_MANIFEST_MEDIA_TYPE = "application/vnd.oci.image.manifest.v1+json" +const val DOCKER_DISTRIBUTION_MANIFEST_V2 = "application/vnd.docker.distribution.manifest.v2+json" + // Content Descriptor const val CONTENT_DESCRIPTOR_MEDIA_TYPE = "application/vnd.oci.descriptor.v1+json" diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt index 5805b903cf..c4b89dd37f 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt @@ -37,7 +37,10 @@ enum class OciMessageCode(private val key: String) : MessageCode { OCI_DELETE_RULES("oci.delete.rules"), OCI_VERSION_NOT_FOUND("oci.version.not.found"), OCI_MANIFEST_INVALID("oci.manifest.invalid"), - OCI_DIGEST_INVALID("oci.digest.invalid") + OCI_DIGEST_INVALID("oci.digest.invalid"), + OCI_MANIFEST_SCHEMA1_NOT_SUPPORT("oci.manifest.schema1.not.support"), + OCI_REMOTE_CONFIGURATION_ERROR("oci.remote.configuration.error"), + OCI_REMOTE_CREDENTIALS_INVALID("oci.remote.credentials.invalid") ; override fun getBusinessCode() = ordinal + 1 override fun getKey() = key diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt new file mode 100644 index 0000000000..e5658715d9 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.extension + +import com.tencent.devops.plugin.api.ExtensionPoint + +/** + * 拉取并存储第三方仓库下的镜像对应package信息扩展点 + */ +interface ImagePackageInfoPullExtension: ExtensionPoint { + + /** + * 拉取第三方镜像仓库package信息,并存储到对应仓库中 + */ + fun queryAndCreateDockerPackageInfo( + context: ImagePackagePullContext + ) +} diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt new file mode 100644 index 0000000000..72d0ba4b88 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.extension + +import java.net.URL + +data class ImagePackagePullContext( + val projectId: String, + val repoName:String, + val remoteUrl: URL, + val userName: String? = null, + val password: String? = null +) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt index 9229cc4541..ad86c29317 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt @@ -60,11 +60,14 @@ open class OciArtifactInfo( const val BOLBS_UPLOAD_SECOND_STEP_URL = "/v2/{projectId}/{repoName}/**/blobs/uploads/{uuid}" // tags get - const val TAGS_URL = "/v2/{projectId}/{repoName}/**/tags/list" + const val TAGS_LIST_SUFFIX = "/tags/list" + // Retrieve a sorted, json list of repositories available in the registry. + const val DOCKER_CATALOG_SUFFIX = "/v2/_catalog" // version详情获取 const val OCI_VERSION_DETAIL = "/version/detail/{projectId}/{repoName}" + // 额外的package或者version 删除接口 const val OCI_PACKAGE_DELETE_URL = "/package/delete/{projectId}/{repoName}" const val OCI_VERSION_DELETE_URL = "/version/delete/{projectId}/{repoName}" @@ -72,6 +75,7 @@ open class OciArtifactInfo( const val OCI_USER_LAYER_SUFFIX = "/layer/{projectId}/{repoName}/**/{id}" const val OCI_USER_REPO_SUFFIX = "/repo/{projectId}/{repoName}" const val OCI_USER_TAG_SUFFIX = "/tag/{projectId}/{repoName}/**" - const val DOCKER_CATALOG_SUFFIX = "_catalog" + + const val OCI_BLOB_NODE_FULLPATH_REFRESH = "/blob/node/refresh" } } diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt deleted file mode 100644 index 762591e162..0000000000 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.tencent.bkrepo.oci.pojo.metadata - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonInclude - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -data class HelmMaintainerMetadata( - val name: String?, - val email: String?, - val url: String? -) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt new file mode 100644 index 0000000000..c6c9363060 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.pojo.remote + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.oci.constant.REQUEST_IMAGE + +data class RemoteRequestProperty( + var url: String, + var imageName: String, + var fullPath: String = StringPool.EMPTY, + var params: String = StringPool.EMPTY, + var type: String = REQUEST_IMAGE +) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/response/ResponseProperty.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/response/ResponseProperty.kt new file mode 100644 index 0000000000..5715b5c5b3 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/response/ResponseProperty.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.pojo.response + +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.oci.pojo.digest.OciDigest + +data class ResponseProperty( + val digest: OciDigest? = null, + val location: String? = null, + val uuid: String? = null, + val range: Long? = null, + val status: HttpStatus? = null, + val contentLength: Int? = null +) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt index 550222ab67..c7b1dbb7c2 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt @@ -27,45 +27,47 @@ package com.tencent.bkrepo.oci.util +import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.oci.constant.OCI_MANIFEST import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest -import org.slf4j.LoggerFactory object OciLocationUtils { - private val logger = LoggerFactory.getLogger(OciLocationUtils::class.java) - fun buildManifestPath(packageName: String, tag: String): String { - return "/$packageName/manifest/$tag/$OCI_MANIFEST" + return buildManifestVersionFolderPath(packageName, tag) + OCI_MANIFEST } - fun buildDigestManifestPathWithReference(packageName: String, reference: String): String { - return buildDigestManifestPath(packageName, OciDigest(reference)) + fun buildManifestVersionFolderPath(packageName: String, tag: String): String { + return buildManifestFolderPath(packageName) +"$tag/" } - fun buildDigestManifestPathWithSha256(packageName: String, sha256: String): String { - return buildDigestManifestPath(packageName, OciDigest.fromSha256(sha256)) + fun buildManifestFolderPath(packageName: String): String { + return "/$packageName/manifest/" } - private fun buildDigestManifestPath(packageName: String, ref: OciDigest): String { - return buildPath(packageName, ref, "manifest") + fun buildDigestManifestPathWithReference(packageName: String, reference: String): String { + return buildDigestManifestPath(packageName, OciDigest(reference)) } - fun buildDigestBlobsPath(packageName: String, digestStr: String): String { - return buildPath(packageName, OciDigest(digestStr), "blobs") + private fun buildDigestManifestPath(packageName: String, ref: OciDigest): String { + return buildPath(packageName, ref, "manifest") } fun buildDigestBlobsPath(packageName: String, ref: OciDigest): String { return buildPath(packageName, ref, "blobs") } + fun buildBlobsFolderPath(packageName: String): String { + return buildPath(packageName, null, "blobs") + } + fun buildDigestBlobsUploadPath(packageName: String, ref: OciDigest): String { return buildPath(packageName, ref, "_uploads") } - private fun buildPath(packageName: String, ref: OciDigest, type: String): String { - return "/$packageName/$type/" + ref.fileName() + private fun buildPath(packageName: String, ref: OciDigest? = null, type: String): String { + return "/$packageName/$type/"+ (ref?.fileName() ?: StringPool.EMPTY) } fun manifestLocation(digest: OciDigest, ociArtifactInfo: OciArtifactInfo): String { @@ -86,6 +88,14 @@ object OciLocationUtils { } } + fun blobVersionPathLocation(reference: String, packageName: String, fileName: String): String { + return blobVersionFolderLocation(reference, packageName)+ fileName + } + + fun blobVersionFolderLocation(reference: String, packageName: String): String { + return "/$packageName/blobs/$reference/" + } + private fun returnPathLocation(digest: OciDigest, ociArtifactInfo: OciArtifactInfo, type: String): String { with(ociArtifactInfo) { return "/$packageName/$type/$digest" diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt index 402247b48f..29004f80ff 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt @@ -30,20 +30,15 @@ package com.tencent.bkrepo.oci.util import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.StreamUtils.readText import com.tencent.bkrepo.common.api.util.readJsonString -import com.tencent.bkrepo.common.api.util.readYamlString -import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.oci.constant.DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1 -import com.tencent.bkrepo.oci.constant.FILE_EXTENSION import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.model.Descriptor import com.tencent.bkrepo.oci.model.ManifestSchema1 import com.tencent.bkrepo.oci.model.ManifestSchema2 import com.tencent.bkrepo.oci.model.SchemaVersion -import com.tencent.bkrepo.oci.pojo.metadata.HelmChartMetadata -import com.tencent.bkrepo.oci.util.DecompressUtil.getArchivesContent import org.apache.logging.log4j.util.Strings import java.io.InputStream @@ -93,15 +88,6 @@ object OciUtils { } } - fun parseChartInputStream(inputStream: InputStream): HelmChartMetadata { - val result = inputStream.getArchivesContent(FILE_EXTENSION) - return result.byteInputStream().readYamlString() - } - - fun convertToMap(chartInfo: HelmChartMetadata): Map { - return chartInfo.toJsonString().readJsonString() - } - fun manifestIterator(manifest: ManifestSchema2): List { val list = mutableListOf() list.add(manifest.config) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index 98722292ea..320e05b323 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -68,6 +68,7 @@ import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest import com.tencent.bkrepo.oci.pojo.response.CatalogResponse +import com.tencent.bkrepo.oci.pojo.response.ResponseProperty import com.tencent.bkrepo.oci.pojo.tags.TagsInfo import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.OciLocationUtils @@ -128,29 +129,13 @@ class OciRegistryLocalRepository( */ override fun onUpload(context: ArtifactUploadContext) { logger.info("Preparing to upload the oci file in repo ${context.artifactInfo.getRepoIdentify()}") - val requestMethod = context.request.method - if (PATCH == requestMethod) { - patchUpload(context) - } else { - val (digest, location) = if (POST == requestMethod) { - postUpload(context) - } else { - putUpload(context) - } - logger.info( - "Artifact ${context.artifactInfo.getArtifactFullPath()} has been uploaded " + - "and will can be accessed in $location" + - " in repo ${context.artifactInfo.getRepoIdentify()}" - ) - if (digest == null || location.isNullOrEmpty()) return - val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) - OciResponseUtils.buildUploadResponse( - domain = domain, - digest = digest, - locationStr = location, - response = context.response - ) - } + val responseProperty = when (context.request.method) { + PATCH -> patchUpload(context) + POST -> postUpload(context) + else -> putUpload(context) + } ?: return + val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) + OciResponseUtils.buildUploadResponse(domain, responseProperty, context.response) } /** @@ -161,27 +146,24 @@ class OciRegistryLocalRepository( * 2:Upload the chunks (PATCH) * 3:Close the session (PUT) */ - private fun patchUpload(context: ArtifactUploadContext) { + private fun patchUpload(context: ArtifactUploadContext): ResponseProperty? { logger.info("Will using patch ways to upload file in repo ${context.artifactInfo.getRepoIdentify()}") - if (context.artifactInfo !is OciBlobArtifactInfo) return + if (context.artifactInfo !is OciBlobArtifactInfo) return null with(context.artifactInfo as OciBlobArtifactInfo) { val range = context.request.getHeader("Content-Range") val length = context.request.contentLength - val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) if (!range.isNullOrEmpty() && length > -1) { logger.info("range $range, length $length, uuid $uuid") val (start, end) = getRangeInfo(range) // 判断要上传的长度是否超长 if (end - start > length - 1) { - OciResponseUtils.buildBlobUploadPatchResponse( - domain = domain, - uuid = uuid!!, - locationStr = OciLocationUtils.blobUUIDLocation(uuid!!, this), - response = HttpContextHolder.getResponse(), + return ResponseProperty( + location = OciLocationUtils.blobUUIDLocation(uuid!!, this), + status = HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE, range = length.toLong(), - status = HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE + uuid = uuid!!, + contentLength = 0 ) - return } } val patchLen = storageService.append( @@ -189,12 +171,12 @@ class OciRegistryLocalRepository( artifactFile = context.getArtifactFile(), storageCredentials = context.repositoryDetail.storageCredentials ) - OciResponseUtils.buildBlobUploadPatchResponse( - domain = domain, + return ResponseProperty( + location = OciLocationUtils.blobUUIDLocation(uuid!!, this), + status = HttpStatus.ACCEPTED, + range = patchLen, uuid = uuid!!, - locationStr = OciLocationUtils.blobUUIDLocation(uuid!!, this), - response = HttpContextHolder.getResponse(), - range = patchLen + contentLength = 0 ) } } @@ -203,7 +185,7 @@ class OciRegistryLocalRepository( * blob 上传,直接使用post * Pushing a blob monolithically :A single POST request */ - private fun postUpload(context: ArtifactUploadContext): Pair { + private fun postUpload(context: ArtifactUploadContext): ResponseProperty? { val artifactFile = context.getArtifactFile() val digest = OciDigest.fromSha256(artifactFile.getFileSha256()) ociOperationService.storeArtifact( @@ -211,13 +193,19 @@ class OciRegistryLocalRepository( artifactFile = artifactFile, storageCredentials = context.storageCredentials ) + val blobLocation = OciLocationUtils.blobLocation(digest, context.artifactInfo as OciArtifactInfo) logger.info( "Artifact ${context.artifactInfo.getArtifactFullPath()} has " + - "been uploaded to ${context.artifactInfo.getArtifactFullPath()}" + + "been uploaded to ${context.artifactInfo.getArtifactFullPath()}, " + + "and will can be accessed in $blobLocation" + " in repo ${context.artifactInfo.getRepoIdentify()}" ) - val blobLocation = OciLocationUtils.blobLocation(digest, context.artifactInfo as OciArtifactInfo) - return Pair(digest, blobLocation) + return ResponseProperty( + digest = digest, + location = blobLocation, + status = HttpStatus.CREATED, + contentLength = 0 + ) } /** @@ -226,14 +214,12 @@ class OciRegistryLocalRepository( * 2 blob POST PATCH with PUT 上传的put模块处理 * 3 manifest PUT上传的逻辑处理 */ - private fun putUpload(context: ArtifactUploadContext): Pair { - if (context.artifactInfo is OciBlobArtifactInfo) { - return putUploadBlob(context) + private fun putUpload(context: ArtifactUploadContext): ResponseProperty? { + return when (context.artifactInfo) { + is OciBlobArtifactInfo -> putUploadBlob(context) + is OciManifestArtifactInfo -> putUploadManifest(context) + else -> null } - if (context.artifactInfo is OciManifestArtifactInfo) { - return putUploadManifest(context) - } - return Pair(null, null) } /** @@ -241,7 +227,7 @@ class OciRegistryLocalRepository( * 1 blob POST with PUT 上传的put模块处理 * 2 blob POST PATCH with PUT 上传的put模块处理 */ - private fun putUploadBlob(context: ArtifactUploadContext): Pair { + private fun putUploadBlob(context: ArtifactUploadContext): ResponseProperty { val artifactInfo = context.artifactInfo as OciBlobArtifactInfo val sha256 = artifactInfo.getDigestHex() val fileInfo = try { @@ -279,19 +265,25 @@ class OciRegistryLocalRepository( } } val digest = OciDigest.fromSha256(fileInfo.sha256) + val blobLocation = OciLocationUtils.blobLocation(digest, artifactInfo) logger.info( "Artifact ${context.artifactInfo.getArtifactFullPath()} " + "has been uploaded to ${context.artifactInfo.getArtifactFullPath()}" + + "and will can be accessed in $blobLocation" + " in repo ${context.artifactInfo.getRepoIdentify()}" ) - val blobLocation = OciLocationUtils.blobLocation(digest, artifactInfo) - return Pair(digest, blobLocation) + return ResponseProperty( + digest = digest, + location = blobLocation, + status = HttpStatus.CREATED, + contentLength = 0 + ) } /** * manifest文件 PUT上传的逻辑处理 */ - private fun putUploadManifest(context: ArtifactUploadContext): Pair { + private fun putUploadManifest(context: ArtifactUploadContext): ResponseProperty { val artifactInfo = context.artifactInfo as OciManifestArtifactInfo val artifactFile = context.getArtifactFile() val digest = OciDigest.fromSha256(artifactFile.getFileSha256()) @@ -300,19 +292,25 @@ class OciRegistryLocalRepository( artifactFile = artifactFile, storageCredentials = context.storageCredentials ) - logger.info( - "Artifact ${context.artifactInfo.getArtifactFullPath()} has been uploaded to ${node!!.fullPath}" + - " in repo ${context.artifactInfo.getRepoIdentify()}" - ) - // 上传manifest文件,同时需要将manifest中对应blob的属性进行补充到blob节点中,同时创建package相关信息 + // 上传manifest文件,同时需要判断manifest中的blob节点是否已经存在,同时创建package相关信息 ociOperationService.updateOciInfo( ociArtifactInfo = artifactInfo, digest = digest, storageCredentials = context.storageCredentials, - nodeDetail = node + nodeDetail = node!! ) val manifestLocation = OciLocationUtils.manifestLocation(digest, artifactInfo) - return Pair(digest, manifestLocation) + logger.info( + "Artifact ${context.artifactInfo.getArtifactFullPath()} has been uploaded to ${node.fullPath}" + + "and will can be accessed in $manifestLocation" + + " in repo ${context.artifactInfo.getRepoIdentify()}" + ) + return ResponseProperty( + digest = digest, + location = manifestLocation, + status = HttpStatus.CREATED, + contentLength = 0 + ) } /** diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index e44181617b..8a6833e65b 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -31,14 +31,21 @@ package com.tencent.bkrepo.oci.artifact.repository +import com.google.common.cache.CacheBuilder import com.tencent.bkrepo.common.api.constant.BEARER_AUTH_PREFIX +import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.HttpHeaders.WWW_AUTHENTICATE import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.util.AuthenticationUtil +import com.tencent.bkrepo.common.api.util.BasicAuthUtils import com.tencent.bkrepo.common.api.util.JsonUtils +import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext @@ -50,31 +57,35 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.stream.artifactStream import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter -import com.tencent.bkrepo.oci.constant.BEARER_REALM +import com.tencent.bkrepo.oci.constant.CATALOG_REQUEST +import com.tencent.bkrepo.oci.constant.DOCKER_DISTRIBUTION_MANIFEST_V2 import com.tencent.bkrepo.oci.constant.DOCKER_LINK import com.tencent.bkrepo.oci.constant.LAST_TAG import com.tencent.bkrepo.oci.constant.MEDIA_TYPE import com.tencent.bkrepo.oci.constant.N import com.tencent.bkrepo.oci.constant.OCI_API_PREFIX +import com.tencent.bkrepo.oci.constant.OCI_FILTER_ENDPOINT import com.tencent.bkrepo.oci.constant.OCI_IMAGE_MANIFEST_MEDIA_TYPE import com.tencent.bkrepo.oci.constant.OciMessageCode -import com.tencent.bkrepo.oci.constant.SCOPE -import com.tencent.bkrepo.oci.constant.SERVICE +import com.tencent.bkrepo.oci.constant.PROXY_URL +import com.tencent.bkrepo.oci.constant.TAG_LIST_REQUEST import com.tencent.bkrepo.oci.exception.OciForbiddenRequestException import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.DOCKER_CATALOG_SUFFIX +import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_LIST_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciBlobArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.pojo.auth.BearerToken import com.tencent.bkrepo.oci.pojo.digest.OciDigest +import com.tencent.bkrepo.oci.pojo.remote.RemoteRequestProperty import com.tencent.bkrepo.oci.pojo.response.CatalogResponse +import com.tencent.bkrepo.oci.pojo.response.OciResponse import com.tencent.bkrepo.oci.pojo.tags.TagsInfo import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.OciLocationUtils import com.tencent.bkrepo.oci.util.OciResponseUtils import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import okhttp3.Credentials import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response @@ -83,6 +94,7 @@ import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.io.InputStream import java.net.URL +import java.util.concurrent.TimeUnit import java.util.regex.Pattern import javax.ws.rs.core.UriBuilder @@ -91,9 +103,15 @@ class OciRegistryRemoteRepository( private val ociOperationService: OciOperationService ) : RemoteRepository() { + private val clientCache = CacheBuilder.newBuilder().maximumSize(100) + .expireAfterWrite(60, TimeUnit.MINUTES).build() + private val tokenCache = CacheBuilder.newBuilder().maximumSize(100) + .expireAfterWrite(60, TimeUnit.MINUTES).build() + + override fun upload(context: ArtifactUploadContext) { with(context) { - val message = "Forbidden to upload chart into a remote repository [$projectId/$repoName]" + val message = "Forbidden to upload artifact into a remote repository [$projectId/$repoName]" logger.warn(message) throw OciForbiddenRequestException(OciMessageCode.OCI_FILE_UPLOAD_FORBIDDEN, "$projectId|$repoName") } @@ -103,8 +121,13 @@ class OciRegistryRemoteRepository( * 下载制品 */ override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? { - return getCacheArtifactResource(context) ?: run { - return doRequest(context) as ArtifactResource? + return if (context.artifactInfo is OciManifestArtifactInfo) { + // 同一镜像tag可能被覆盖更新,对于manifest.json文件每次都去远端拉取 + doRequest(context) as ArtifactResource? + } else { + getCacheArtifactResource(context) ?: run { + doRequest(context) as ArtifactResource? + } } } @@ -122,35 +145,68 @@ class OciRegistryRemoteRepository( */ private fun doRequest(context: ArtifactContext): Any? { val remoteConfiguration = context.getRemoteConfiguration() - val httpClient = createHttpClient(remoteConfiguration, false) - val downloadUrl = createRemoteDownloadUrl(context) + val httpClient = clientCache.getIfPresent(remoteConfiguration) ?: run { + clientCache.put(remoteConfiguration, createHttpClient(remoteConfiguration, false)) + clientCache.getIfPresent(remoteConfiguration) + } + val property = getRemoteRequestProperty(context) + val downloadUrl = createRemoteDownloadUrl(context, property) logger.info("Remote request $downloadUrl will be sent") - val request = buildRequest(downloadUrl, remoteConfiguration) - val response = httpClient.newCall(request).execute() - var responseWithAuth: Response? = null + val tokenKey = buildTokenCacheKey( + context.getStringAttribute(PROXY_URL)!!,remoteConfiguration.credentials.username, property.imageName + ) + val request = buildRequest(downloadUrl, remoteConfiguration, tokenCache.getIfPresent(tokenKey)) try { - if (response.isSuccessful) return onResponse(context, response) - // 针对返回401进行token获取 - val token = getAuthenticationCode(response, remoteConfiguration, httpClient) - if (token.isNullOrBlank()) return null - val requestWithAuth = buildRequest( - url = downloadUrl, - configuration = remoteConfiguration, - addBasicInterceptor = false, - token = token - ) - responseWithAuth = httpClient - .newCall(requestWithAuth) - .execute() + httpClient!!.newCall(request).execute().use { + if (it.isSuccessful) return onResponse(context, it) + if (it.code != HttpStatus.UNAUTHORIZED.value) { + logger.warn("response code is ${it.code} for url $downloadUrl") + return null + } + return doRequestWithToken( + context = context, + wwwAuthenticate = it.header(WWW_AUTHENTICATE), + remoteConfiguration = remoteConfiguration, + imageName = property.imageName, + tokenKey = tokenKey, + downloadUrl = downloadUrl + ) + } + } catch (e: Exception) { + logger.error("Error occurred while sending request $downloadUrl", e) + throw NodeNotFoundException(downloadUrl) + } + } + + /** + * 当返回401时,按照docker标准协议去拉取token,然后进行文件下载 + */ + private fun doRequestWithToken( + context: ArtifactContext, + wwwAuthenticate: String?, + remoteConfiguration: RemoteConfiguration, + imageName: String, + tokenKey: String, + downloadUrl: String, + ): Any? { + // 针对返回401进行token获取 + val proxyUrl = context.getStringAttribute(PROXY_URL)!! + val token = getAuthenticationCode(proxyUrl, wwwAuthenticate, remoteConfiguration, imageName) ?: return null + tokenCache.put(tokenKey, token) + val requestWithToken = buildRequest( + url = downloadUrl, + configuration = remoteConfiguration, + addBasicInterceptor = false, + token = token + ) + clientCache.getIfPresent(remoteConfiguration)!!.newCall(requestWithToken).execute().use {responseWithAuth -> return if (checkResponse(responseWithAuth)) { onResponse(context, responseWithAuth) } else null - } finally { - response.body?.close() - responseWithAuth?.body?.close() } } + private fun onResponse(context: ArtifactContext, response: Response): Any? { if (context is ArtifactDownloadContext) { return onDownloadResponse(context, response) @@ -171,11 +227,15 @@ class OciRegistryRemoteRepository( addBasicInterceptor: Boolean = true ): Request { val requestBuilder = Request.Builder().url(url) - if (addBasicInterceptor) { + if (addBasicInterceptor && token.isNullOrEmpty()) { requestBuilder.addInterceptor(configuration) } else { token?.let { requestBuilder.header(HttpHeaders.AUTHORIZATION, token) } } + // 拉取第三方仓库时,默认会返回v1版本的镜像格式 + if (url.contains("/manifests/")) { + requestBuilder.header(HttpHeaders.ACCEPT, DOCKER_DISTRIBUTION_MANIFEST_V2) + } return requestBuilder.build() } @@ -183,7 +243,7 @@ class OciRegistryRemoteRepository( val username = configuration.credentials.username val password = configuration.credentials.password if (username != null && password != null) { - val credentials = Credentials.basic(username, password) + val credentials = BasicAuthUtils.encode(username, password) this.header(HttpHeaders.AUTHORIZATION, credentials) } return this @@ -193,110 +253,158 @@ class OciRegistryRemoteRepository( * 生成远程构件下载url */ override fun createRemoteDownloadUrl(context: ArtifactContext): String { - val configuration = context.getRemoteConfiguration() - if (context.artifactInfo is OciBlobArtifactInfo) { - val artifactInfo = context.artifactInfo as OciBlobArtifactInfo - return createUrl( - url = configuration.url, - fullPath = OciLocationUtils.blobPathLocation(artifactInfo.getDigest(), artifactInfo), - params = StringPool.EMPTY - ) - } - if (context.artifactInfo is OciManifestArtifactInfo) { - val artifactInfo = context.artifactInfo as OciManifestArtifactInfo - return createUrl( - url = configuration.url, - fullPath = OciLocationUtils.manifestPathLocation(artifactInfo.reference, artifactInfo), - params = StringPool.EMPTY - ) + val property = getRemoteRequestProperty(context) + return createRemoteDownloadUrl(context, property) + } + + fun createRemoteDownloadUrl(context: ArtifactContext, property: RemoteRequestProperty): String { + return when (property.type) { + CATALOG_REQUEST -> createCatalogUrl(property) + TAG_LIST_REQUEST -> createTagListUrl(property) + else -> createUrl(property) } - if (context.artifactInfo is OciTagArtifactInfo) { - val artifactInfo = context.artifactInfo as OciTagArtifactInfo - if (artifactInfo.packageName.isBlank()) { - val (_, params) = createParamsForTagList(context) - return createCatalogUrl(configuration.url, params) - } else { - val (fullPath, params) = createParamsForTagList(context) - return createUrl(configuration.url, fullPath, params) + } + + /** + * 获取不同情况下对应的请求属性 + */ + private fun getRemoteRequestProperty(context: ArtifactContext): RemoteRequestProperty { + val configuration = context.getRemoteConfiguration() + val url = UrlFormatter.addProtocol(configuration.url).toString() + context.putAttribute(PROXY_URL, url) + return when (context.artifactInfo) { + is OciBlobArtifactInfo -> { + val artifactInfo = context.artifactInfo as OciBlobArtifactInfo + RemoteRequestProperty( + url = url, + fullPath = OciLocationUtils.blobPathLocation(artifactInfo.getDigest(), artifactInfo), + imageName = artifactInfo.packageName + ) } + is OciTagArtifactInfo -> { + val artifactInfo = context.artifactInfo as OciTagArtifactInfo + if (artifactInfo.packageName.isBlank()) { + val (_, params) = createParamsForTagList(context) + RemoteRequestProperty( + url = url, + params = params, + type = CATALOG_REQUEST, + imageName = StringPool.EMPTY + ) + } else { + val (fullPath, params) = createParamsForTagList(context) + RemoteRequestProperty( + url = url, + fullPath = fullPath, + params = params, + type = TAG_LIST_REQUEST, + imageName = artifactInfo.packageName + ) + } + } + is OciManifestArtifactInfo -> { + val artifactInfo = context.artifactInfo as OciManifestArtifactInfo + RemoteRequestProperty( + url = url, + fullPath = OciLocationUtils.manifestPathLocation(artifactInfo.reference, artifactInfo), + imageName = artifactInfo.packageName + ) + } + else -> RemoteRequestProperty(url = url, imageName = StringPool.EMPTY) } - return createUrl(configuration.url) } /** * 拼接url */ - private fun createUrl(url: String, fullPath: String = StringPool.EMPTY, params: String = StringPool.EMPTY): String { - val baseUrl = URL(url) - val v2Url = URL(baseUrl, "/v2" + baseUrl.path) - return UrlFormatter.format(v2Url.toString(), fullPath, params) + private fun createUrl(property: RemoteRequestProperty): String { + with(property) { + val baseUrl = URL(url) + val v2Url = URL(baseUrl, OCI_FILTER_ENDPOINT + baseUrl.path) + return UrlFormatter.format(v2Url.toString(), fullPath, params) + } } /** * 拼接catalog url */ - private fun createCatalogUrl(url: String, params: String = StringPool.EMPTY): String { - val baseUrl = URL(url) - val builder = UriBuilder.fromPath(OCI_API_PREFIX) - .host(baseUrl.host).scheme(baseUrl.protocol) - .path(DOCKER_CATALOG_SUFFIX) - .queryParam(params) - return builder.build().toString() + private fun createCatalogUrl(property: RemoteRequestProperty): String { + with(property) { + return UrlFormatter.buildUrl(url, DOCKER_CATALOG_SUFFIX, params) + } + } + + /** + * 拼接tag list url + */ + private fun createTagListUrl(property: RemoteRequestProperty): String { + with(property) { + val url = UriBuilder.fromUri(url) + .path(OCI_API_PREFIX) + .path(imageName) + .path(TAGS_LIST_SUFFIX) + .build().toString() + return UrlFormatter.addParams(url, params) + } } private fun getAuthenticationCode( - response: Response, + proxyUrl: String, + wwwAuthenticate: String?, configuration: RemoteConfiguration, - httpClient: OkHttpClient + imageName: String ): String? { - if (response.code != HttpStatus.UNAUTHORIZED.value) { + if (wwwAuthenticate.isNullOrBlank() || !wwwAuthenticate.startsWith(BEARER_AUTH_PREFIX)) { + logger.warn("response wwwAuthenticate header $wwwAuthenticate is illegal") return null } - val wwwAuthenticate = response.header(WWW_AUTHENTICATE) - if (wwwAuthenticate.isNullOrBlank() || !wwwAuthenticate.startsWith(BEARER_AUTH_PREFIX)) { + val scope = getScope(proxyUrl, imageName) + val authProperty = AuthenticationUtil.parseWWWAuthenticateHeader(wwwAuthenticate, scope) + if (authProperty == null) { + logger.warn("Auth url can not be parsed from header $wwwAuthenticate!") return null } - val url = parseWWWAuthenticateHeader(wwwAuthenticate) - logger.info("The url for authenticating is $url") - if (url.isNullOrEmpty()) return null + val urlStr = AuthenticationUtil.buildAuthenticationUrl(authProperty, configuration.credentials.username) + logger.info("The url for authentication is $urlStr") + if (urlStr.isNullOrEmpty()) return null val request = buildRequest( - url = url, + url = urlStr, configuration = configuration, - addBasicInterceptor = false + addBasicInterceptor = true ) - val tokenResponse = httpClient.newCall(request).execute() - try { - if (!tokenResponse.isSuccessful) return null - val body = tokenResponse.body!! - val artifactFile = createTempFile(body) - val size = artifactFile.getSize() - val artifactStream = artifactFile.getInputStream().artifactStream(Range.full(size)) - artifactFile.delete() - val bearerToken = JsonUtils.objectMapper.readValue(artifactStream, BearerToken::class.java) - return "Bearer ${bearerToken.token}" - } finally { - tokenResponse.body?.close() + clientCache.getIfPresent(configuration)!!.newCall(request).execute().use { + if (!it.isSuccessful) { + val error = try { + JsonUtils.objectMapper.readValue(it.body!!.byteStream(), OciResponse::class.java).toJsonString() + } catch (ignore: Exception) { + StringPool.EMPTY + } + val errMsg = "Could not get token from auth service," + + " code is ${it.code} and response is $error" + logger.warn(errMsg) + throw ErrorCodeException( + OciMessageCode.OCI_REMOTE_CREDENTIALS_INVALID, + errMsg + ) + } + try { + val bearerToken = JsonUtils.objectMapper.readValue(it.body!!.byteStream(), BearerToken::class.java) + return "Bearer ${bearerToken.token}" + } catch (e: Exception) { + throw ErrorCodeException( + OciMessageCode.OCI_REMOTE_CONFIGURATION_ERROR, + "Could not get token from auth service, please check your remote configuration!" + ) + } } } - /** - * 解析返回头中的WWW_AUTHENTICATE字段, 只针对为Bearer realm - */ - private fun parseWWWAuthenticateHeader(wwwAuthenticate: String): String? { - val map: MutableMap = mutableMapOf() - return try { - val params = wwwAuthenticate.split(",") - params.forEach { - val param = it.split("=") - val name = param.first() - val value = param.last().removeSurrounding("\"") - map[name] = value - } - "${map[BEARER_REALM]}?$SERVICE=${map[SERVICE]}&$SCOPE=${map[SCOPE]}" - } catch (e: Exception) { - logger.warn("Parsing wwwAuthenticate header error: ${e.message}") - null - } + + private fun getScope(remoteUrl: String, imageName: String): String { + val baseUrl = URL(remoteUrl) + val target = baseUrl.path.removePrefix(StringPool.SLASH) + .removeSuffix(StringPool.SLASH) + StringPool.SLASH + imageName + return "repository:$target:pull" } private fun createParamsForTagList(context: ArtifactContext): Pair { @@ -331,7 +439,7 @@ class OciRegistryRemoteRepository( inputStream = artifactStream, artifactName = context.artifactInfo.getResponseName(), node = node, - channel = ArtifactChannel.LOCAL + channel = ArtifactChannel.PROXY ) return buildResponse( cacheNode = node, @@ -412,13 +520,14 @@ class OciRegistryRemoteRepository( // 针对manifest文件获取会通过tag或manifest获取,避免重复创建 fullPath?.let { val node = nodeClient.getNodeDetail(ociArtifactInfo.projectId, ociArtifactInfo.repoName, fullPath).data - if (node != null) return node + if (node != null && artifactFile.getFileSha256() == node.sha256) return node } + val url = context.getStringAttribute(PROXY_URL) var nodeDetail = ociOperationService.storeArtifact( ociArtifactInfo = ociArtifactInfo, artifactFile = artifactFile, storageCredentials = context.storageCredentials, - proxyUrl = configuration.url + proxyUrl = url ) // 针对manifest文件需要更新metadata if (context.artifactInfo is OciManifestArtifactInfo) { @@ -435,7 +544,7 @@ class OciRegistryRemoteRepository( private fun updateManifestAndBlob(context: ArtifactDownloadContext, nodeDetail: NodeDetail) { with(context.artifactInfo as OciManifestArtifactInfo) { val digest = OciDigest.fromSha256(nodeDetail.sha256!!) - // 上传manifest文件,同时需要将manifest中对应blob的属性进行补充到blob节点中,同时创建package相关信息 + // 上传manifest文件,同时创建package相关信息 ociOperationService.updateOciInfo( ociArtifactInfo = this, digest = digest, @@ -506,6 +615,11 @@ class OciRegistryRemoteRepository( return n } + private fun buildTokenCacheKey(remoteUrl: String, userName: String?, imageName: String): String { + val scope = getScope(remoteUrl, imageName) + return "$remoteUrl${CharPool.COLON}$scope${CharPool.COLON}$userName" + } + companion object { val logger: Logger = LoggerFactory.getLogger(OciRegistryRemoteRepository::class.java) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt index 0dbd8a7f01..d5830211a3 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt @@ -38,13 +38,12 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.resolve.path.ArtifactInfoResolver import com.tencent.bkrepo.common.artifact.resolve.path.Resolver import com.tencent.bkrepo.oci.constant.OCI_TAG -import com.tencent.bkrepo.oci.constant.USER_API_PREFIX import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.DOCKER_CATALOG_SUFFIX +import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_LIST_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo -import io.undertow.servlet.spec.HttpServletRequestImpl -import javax.servlet.http.HttpServletRequest import org.springframework.stereotype.Component import org.springframework.web.servlet.HandlerMapping +import javax.servlet.http.HttpServletRequest @Component @Resolver(OciTagArtifactInfo::class) @@ -58,17 +57,14 @@ class OciTagArtifactInfoResolver : ArtifactInfoResolver { ): ArtifactInfo { val requestURL = ArtifactContextHolder.getUrlPath(this.javaClass.name)!! return when { - requestURL.contains(TAG_PREFIX) -> { + requestURL.contains(TAGS_LIST_SUFFIX) -> { val requestUrl = request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE).toString() - val packageName = requestUrl.removePrefix("$USER_API_PREFIX/tag/$projectId/$repoName/") + val packageName = requestUrl.removePrefix("/$projectId/$repoName/v2/").removeSuffix(TAGS_LIST_SUFFIX) validate(packageName) val tag = request.getParameter(OCI_TAG) ?: StringPool.EMPTY OciTagArtifactInfo(projectId, repoName, packageName, tag) } requestURL.contains(DOCKER_CATALOG_SUFFIX) -> { - val params = (request as HttpServletRequestImpl).queryParameters - val projectId = params?.get("projectId")?.first ?: StringPool.EMPTY - val repoName = params?.get("repoName")?.first ?: StringPool.EMPTY OciTagArtifactInfo(projectId, repoName, StringPool.EMPTY, StringPool.EMPTY) } else -> { @@ -89,6 +85,5 @@ class OciTagArtifactInfoResolver : ArtifactInfoResolver { companion object { const val PACKAGE_NAME_PATTERN = "[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*" - const val TAG_PREFIX = "/ext/tag/" } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt index aa7bbe8695..8aaaccd54c 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt @@ -28,9 +28,12 @@ package com.tencent.bkrepo.oci.controller.service import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.event.repo.RepoCreatedEvent +import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.oci.api.OciClient import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao +import com.tencent.bkrepo.oci.listener.base.EventExecutor import com.tencent.bkrepo.oci.model.TOciReplicationRecord import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.third.OciReplicationRecordInfo @@ -42,8 +45,9 @@ import org.springframework.web.bind.annotation.RestController @RestController class OciPackageController( private val operationService: OciOperationService, - private val ociReplicationRecordDao: OciReplicationRecordDao -): OciClient { + private val ociReplicationRecordDao: OciReplicationRecordDao, + private val eventExecutor: EventExecutor + ): OciClient { override fun packageCreate(record: OciReplicationRecordInfo): Response { with(record) { val ociArtifactInfo = OciManifestArtifactInfo( @@ -65,4 +69,32 @@ class OciPackageController( return ResponseBuilder.success() } } + + override fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String): Response { + eventExecutor.submit(RepoCreatedEvent( + projectId = projectId, + repoName = repoName, + userId = SecurityUtils.getUserId() + )) + return ResponseBuilder.success() + } + + override fun blobPathRefresh( + projectId: String, repoName: String, packageName: String, version: String + ): Response { + return ResponseBuilder.success( + operationService.refreshBlobNode( + projectId = projectId, + repoName = repoName, + pName = packageName, + pVersion = version + )) + } + + override fun deleteBlobsFolderAfterRefreshed( + projectId: String, repoName: String, packageName: String + ): Response { + operationService.deleteBlobsFolderAfterRefreshed(projectId, repoName, packageName) + return ResponseBuilder.success() + } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt index b7b8a0f4cc..d70aa77742 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt @@ -34,11 +34,11 @@ package com.tencent.bkrepo.oci.controller.user import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.security.permission.Permission -import com.tencent.bkrepo.oci.config.OciProperties import com.tencent.bkrepo.oci.constant.DOCKER_API_VERSION import com.tencent.bkrepo.oci.constant.DOCKER_HEADER_API_VERSION import com.tencent.bkrepo.oci.constant.DOCKER_LINK -import com.tencent.bkrepo.oci.constant.OCI_FILTER_ENDPOINT +import com.tencent.bkrepo.oci.constant.OCI_PROJECT_ID +import com.tencent.bkrepo.oci.constant.OCI_REPO_NAME import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.DOCKER_CATALOG_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.service.OciCatalogService @@ -47,29 +47,25 @@ import org.springframework.http.HttpHeaders import org.springframework.http.HttpStatus import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController @RestController -@RequestMapping(OCI_FILTER_ENDPOINT) -class CatalogController( - private val catalogService: OciCatalogService, - private val ociProperties: OciProperties -) { +class CatalogController(private val catalogService: OciCatalogService) { /** * 返回仓库下所有image名列表 */ - @GetMapping(DOCKER_CATALOG_SUFFIX) + @GetMapping("/{projectId}/{repoName}$DOCKER_CATALOG_SUFFIX") @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) fun list( artifactInfo: OciTagArtifactInfo, - @RequestParam(required = true) - @ApiParam(required = true) + @PathVariable + @ApiParam(value = OCI_PROJECT_ID, required = true) projectId: String, - @RequestParam(required = true) - @ApiParam(required = true) + @PathVariable + @ApiParam(value = OCI_REPO_NAME, required = true) repoName: String, @RequestParam(required = false) @ApiParam(value = "n", required = false) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt index 9a5e57ce73..0f0f3a8ac9 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt @@ -106,7 +106,7 @@ class OciBlobController( } /** - * 删除manifest文件 + * 删除blob文件 * 只能通过digest删除 */ @DeleteMapping(BOLBS_URL) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt index e4c265832b..c005855fdc 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt @@ -30,9 +30,8 @@ package com.tencent.bkrepo.oci.controller.user import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.security.permission.Permission -import com.tencent.bkrepo.oci.config.OciProperties import com.tencent.bkrepo.oci.constant.DOCKER_LINK -import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_URL +import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_LIST_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.pojo.tags.TagsInfo import com.tencent.bkrepo.oci.service.OciTagService @@ -48,15 +47,12 @@ import org.springframework.web.bind.annotation.RestController * oci tag controller */ @RestController -class OciTagController( - private val ociTagService: OciTagService, - private val ociProperties: OciProperties -) { +class OciTagController(private val ociTagService: OciTagService) { /** * 获取blob对应的tag信息 */ @Permission(type = ResourceType.REPO, action = PermissionAction.READ) - @RequestMapping(TAGS_URL, method = [RequestMethod.GET]) + @RequestMapping("/{projectId}/{repoName}/**$TAGS_LIST_SUFFIX", method = [RequestMethod.GET]) fun getTagList( artifactInfo: OciTagArtifactInfo, @RequestParam n: Int?, diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt index 8394e2d629..3d24080a5f 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt @@ -56,7 +56,6 @@ import com.tencent.bkrepo.oci.service.OciOperationService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam -import javax.servlet.http.HttpServletRequest import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable @@ -65,6 +64,7 @@ import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController import org.springframework.web.servlet.HandlerMapping +import javax.servlet.http.HttpServletRequest @Suppress("MVCPathVariableInspection") @Api("oci产品接口") diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt index e743922882..c8bcdf504e 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.oci.listener.base import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.common.artifact.event.base.EventType import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel @@ -39,10 +40,12 @@ import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.Logger import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component import java.util.concurrent.Future import java.util.concurrent.ThreadPoolExecutor -open class EventExecutor( +@Component +class EventExecutor( open val nodeClient: NodeClient, open val repositoryClient: RepositoryClient, open val ociOperationService: OciOperationService @@ -57,7 +60,7 @@ open class EventExecutor( ): Future { return threadPoolExecutor.submit { try { - replicationEventHandler(event) + eventHandler(event) true } catch (exception: Throwable) { logger.warn("Error occurred while executing the oci event: $exception") @@ -66,6 +69,16 @@ open class EventExecutor( } } + private fun eventHandler(event: ArtifactEvent) { + when (event.type) { + EventType.REPLICATION_THIRD_PARTY -> replicationEventHandler(event) + EventType.REPO_CREATED, EventType.REPO_REFRESHED, EventType.REPO_UPDATED -> { + ociOperationService.getPackagesFromThirdPartyRepo(event.projectId, event.repoName) + } + else -> throw UnsupportedOperationException() + } + } + private fun replicationEventHandler(event: ArtifactEvent) { with(event) { val packageName = event.data["packageName"].toString() diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmChartMetadata.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt similarity index 53% rename from src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmChartMetadata.kt rename to src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt index 011fa99f2b..753b5fd69b 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmChartMetadata.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt @@ -25,44 +25,43 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.oci.pojo.metadata +package com.tencent.bkrepo.oci.listener.consumer -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonInclude -import com.github.zafarkhaja.semver.Version +import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.common.artifact.event.base.EventType +import com.tencent.bkrepo.oci.listener.base.EventExecutor +import org.slf4j.LoggerFactory +import org.springframework.messaging.Message +import org.springframework.stereotype.Component +import java.util.function.Consumer -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -data class HelmChartMetadata( - var apiVersion: String?, - var appVersion: String?, - var created: String?, - var deprecated: Boolean?, - var description: String?, - var digest: String?, - var engine: String?, - var home: String?, - var icon: String?, - var keywords: List = emptyList(), - var maintainers: List = emptyList(), - var name: String, - var sources: List = emptyList(), - var urls: List = emptyList(), - var version: String, - var type: String?, - var annotations: Map? -) : Comparable { +/** + * 构件事件消费者,用于实时同步 + * 对应destination为对应ArtifactEvent.topic + */ +@Component("remoteOciRepo") +class RemoteImageRepoEventConsumer( + private val eventExecutor: EventExecutor +) : Consumer>{ + + /** + * 允许接收的事件类型 + */ + private val acceptTypes = setOf( + EventType.REPO_CREATED, + EventType.REPO_UPDATED, + EventType.REPO_REFRESHED + ) - override fun compareTo(other: HelmChartMetadata): Int { - val result = this.name.compareTo(other.name) - return if (result != 0) { - result - } else { - try { - Version.valueOf(other.version).compareWithBuildsTo(Version.valueOf(this.version)) - } catch (ignored: Exception) { - other.version.compareTo(this.version) - } + override fun accept(message: Message) { + if (!acceptTypes.contains(message.payload.type)) { + return } + logger.info("current repo operation message header is ${message.headers}") + eventExecutor.submit(message.payload) + } + + companion object { + private val logger = LoggerFactory.getLogger(RemoteImageRepoEventConsumer::class.java) } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt index 5ae0f39e02..7a7e928732 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt @@ -29,9 +29,6 @@ package com.tencent.bkrepo.oci.listener.consumer import com.tencent.bkrepo.common.artifact.event.replication.ThirdPartyReplicationEvent import com.tencent.bkrepo.oci.listener.base.EventExecutor -import com.tencent.bkrepo.oci.service.OciOperationService -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.springframework.context.event.EventListener import org.springframework.stereotype.Component @@ -40,15 +37,13 @@ import org.springframework.stereotype.Component */ @Component class ReplicationEventListener( - override val nodeClient: NodeClient, - override val repositoryClient: RepositoryClient, - override val ociOperationService: OciOperationService -): EventExecutor(nodeClient, repositoryClient, ociOperationService) { + private val eventExecutor: EventExecutor +){ /** * 第三方同步事件处理 */ @EventListener(ThirdPartyReplicationEvent::class) fun handle(event: ThirdPartyReplicationEvent) { - submit(event) + eventExecutor.submit(event) } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt index 9dad95d058..bad477b59f 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt @@ -30,9 +30,6 @@ package com.tencent.bkrepo.oci.listener.consumer import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.common.artifact.event.base.EventType import com.tencent.bkrepo.oci.listener.base.EventExecutor -import com.tencent.bkrepo.oci.service.OciOperationService -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.messaging.Message import org.springframework.stereotype.Component @@ -45,12 +42,8 @@ import java.util.function.Consumer */ @Component("thirdPartyReplication") class ThirdPartyReplicationEventConsumer( - override val nodeClient: NodeClient, - override val repositoryClient: RepositoryClient, - override val ociOperationService: OciOperationService -) : - Consumer>, - EventExecutor(nodeClient, repositoryClient, ociOperationService) { + private val eventExecutor: EventExecutor +) : Consumer> { /** * 允许接收的事件类型 @@ -64,7 +57,7 @@ class ThirdPartyReplicationEventConsumer( return } logger.info("current third party replication message header is ${message.headers}") - submit(message.payload) + eventExecutor.submit(message.payload) } companion object { diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index ad696b23e6..dd63c325d3 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -29,6 +29,7 @@ package com.tencent.bkrepo.oci.service import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel +import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo @@ -39,44 +40,10 @@ import com.tencent.bkrepo.oci.pojo.response.OciImageResult import com.tencent.bkrepo.oci.pojo.response.OciTagResult import com.tencent.bkrepo.oci.pojo.user.PackageVersionInfo import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import javax.servlet.http.HttpServletRequest interface OciOperationService { - /** - * 保存节点元数据 - */ - fun saveMetaData( - projectId: String, - repoName: String, - fullPath: String, - metadata: MutableMap, - userId: String - ) - - /** - * 当mediatype为CHART_LAYER_MEDIA_TYPE,需要解析chart.yaml文件 - */ - fun loadArtifactInput( - chartDigest: String?, - projectId: String, - repoName: String, - packageName: String, - version: String, - storageCredentials: StorageCredentials? - ): Map? - - /** - * 需要将blob中相关metadata写进package version中 - */ - fun updatePackageInfo( - ociArtifactInfo: OciArtifactInfo, - packageKey: String, - appVersion: String? = null, - description: String? = null - ) - /** * 查询包版本详情 */ @@ -121,11 +88,6 @@ interface OciOperationService { manifestPath: String, ): Boolean - /** - * 当使用追加上传时,文件已存储,只需存储节点信息 - */ - fun createNode(request: NodeCreateRequest, storageCredentials: StorageCredentials?): NodeDetail - /** * 保存文件内容(当使用追加上传时,文件已存储,只需存储节点信息) * 特殊:对于manifest文件,node存tag @@ -145,13 +107,14 @@ interface OciOperationService { fun getNodeFullPath(artifactInfo: OciArtifactInfo): String? /** - * 根据sha256值获取对应的node fullpath,md5,size + * 根据sha256值和path获取对应的node fullpath,md5,size */ fun getNodeByDigest( projectId: String, repoName: String, - digestStr: String - ): NodeProperty + digestStr: String, + path: String? = null + ): NodeProperty? /** * 针对老的docker仓库的数据做兼容性处理 @@ -193,4 +156,31 @@ interface OciOperationService { pageSize: Int, tag: String? ): OciTagResult + + /** + * 拉取第三方镜像仓库package信息 + */ + fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String) + + /** + * oci blob 路径调整,由/packageName/blobs/XXX -> /packageName/blobs/version/XXX + */ + fun refreshBlobNode( + projectId: String, + repoName: String, + pName: String, + pVersion: String, + userId: String = SecurityUtils.getUserId() + ): Boolean + + + /** + * 当package下所有版本的blob路径都刷新到新的路径下后,删除/packageName/blobs目录 + */ + fun deleteBlobsFolderAfterRefreshed( + projectId: String, + repoName: String, + pName: String, + userId: String = SecurityUtils.getUserId() + ) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt index 07d76fa503..53646481db 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt @@ -43,17 +43,22 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadConte import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_KEY +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_VALUE import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.REPO_TYPE import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.pojo.artifact.OciBlobArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest +import com.tencent.bkrepo.oci.pojo.response.ResponseProperty import com.tencent.bkrepo.oci.service.OciBlobService import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.ObjectBuildUtils import com.tencent.bkrepo.oci.util.OciLocationUtils import com.tencent.bkrepo.oci.util.OciResponseUtils +import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -61,6 +66,7 @@ import org.springframework.stereotype.Service class OciBlobServiceImpl( private val storage: StorageService, private val repoClient: RepositoryClient, + private val nodeClient: NodeClient, private val ociOperationService: OciOperationService, private val permissionManager: PermissionManager ) : OciBlobService { @@ -95,10 +101,15 @@ class OciBlobServiceImpl( logger.info("Will obtain uuid for uploading blobs in repo ${artifactInfo.getRepoIdentify()}.") val uuidCreated = startAppend(this) val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) - OciResponseUtils.buildBlobUploadUUIDResponse( + val responseProperty = ResponseProperty( + uuid = uuidCreated, + location = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), + status = HttpStatus.ACCEPTED, + contentLength = 0 + ) + OciResponseUtils.buildUploadResponse( domain, - uuidCreated, - OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), + responseProperty, HttpContextHolder.getResponse() ) } else { @@ -120,50 +131,59 @@ class OciBlobServiceImpl( repoName = mountRepoName ) } catch (e: ErrorCodeException) { - val uuidCreated = startAppend(this) - OciResponseUtils.buildBlobMountResponse( - domain = domain, - locationStr = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), - status = HttpStatus.ACCEPTED, - response = HttpContextHolder.getResponse() - ) + buildSessionIdLocationForUpload(this, domain) return } } val nodeProperty = ociOperationService.getNodeByDigest( mountProjectId, mountRepoName, ociDigest.toString() - ) - if (nodeProperty.fullPath == null) { + ) ?: run { logger.warn("Could not find $ociDigest in repo $mountProjectId|$mountRepoName to mount") - val uuidCreated = startAppend(this) - OciResponseUtils.buildBlobMountResponse( - domain = domain, - locationStr = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), - status = HttpStatus.ACCEPTED, - response = HttpContextHolder.getResponse() - ) + buildSessionIdLocationForUpload(this, domain) return } + // 用于新版本 blobs 路径区分 + val metadata: MutableList = mutableListOf( + MetadataModel(key = BLOB_PATH_VERSION_KEY, value = BLOB_PATH_VERSION_VALUE, system = true) + ) val nodeCreateRequest = ObjectBuildUtils.buildNodeCreateRequest( projectId = projectId, repoName = repoName, size = nodeProperty.size!!, sha256 = ociDigest.hex, fullPath = OciLocationUtils.buildDigestBlobsPath(packageName, ociDigest), - md5 = nodeProperty.md5!! + md5 = nodeProperty.md5!!, + metadata = metadata ) - val repoDetail = repoClient.getRepoDetail(projectId, repoName).data - ociOperationService.createNode(nodeCreateRequest, repoDetail!!.storageCredentials) + nodeClient.createNode(nodeCreateRequest) val blobLocation = OciLocationUtils.blobLocation(ociDigest, this) - OciResponseUtils.buildBlobMountResponse( + val responseProperty = ResponseProperty( + location = blobLocation, + status = HttpStatus.CREATED + ) + OciResponseUtils.buildUploadResponse( domain = domain, - locationStr = blobLocation, - status = HttpStatus.CREATED, + responseProperty = responseProperty, response = HttpContextHolder.getResponse() ) } } + private fun buildSessionIdLocationForUpload(artifactInfo: OciBlobArtifactInfo, domain: String) { + val uuidCreated = startAppend(artifactInfo) + val responseProperty = ResponseProperty( + uuid = uuidCreated, + location = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), + status = HttpStatus.ACCEPTED, + contentLength = 0 + ) + OciResponseUtils.buildUploadResponse( + domain = domain, + responseProperty = responseProperty, + response = HttpContextHolder.getResponse() + ) + } + private fun splitRepoInfo(from: String?): Pair? { if (from.isNullOrEmpty()) return null val values = from.split(CharPool.SLASH) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index b810b5a7a5..9bb3516df1 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -32,14 +32,19 @@ import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.StreamUtils.readText import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE +import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration +import com.tencent.bkrepo.common.artifact.pojo.configuration.composite.CompositeConfiguration +import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils @@ -47,10 +52,10 @@ import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.config.OciProperties -import com.tencent.bkrepo.oci.constant.APP_VERSION -import com.tencent.bkrepo.oci.constant.CHART_LAYER_MEDIA_TYPE +import com.tencent.bkrepo.oci.constant.BLOB_PATH_REFRESHED_KEY +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_KEY +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_VALUE import com.tencent.bkrepo.oci.constant.DESCRIPTION -import com.tencent.bkrepo.oci.constant.DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1 import com.tencent.bkrepo.oci.constant.DOWNLOADS import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_BY import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_DATE @@ -68,6 +73,8 @@ import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.exception.OciFileNotFoundException import com.tencent.bkrepo.oci.exception.OciVersionNotFoundException +import com.tencent.bkrepo.oci.extension.ImagePackageInfoPullExtension +import com.tencent.bkrepo.oci.extension.ImagePackagePullContext import com.tencent.bkrepo.oci.model.Descriptor import com.tencent.bkrepo.oci.model.ManifestSchema2 import com.tencent.bkrepo.oci.model.TOciReplicationRecord @@ -84,6 +91,7 @@ import com.tencent.bkrepo.oci.pojo.user.PackageVersionInfo import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.ObjectBuildUtils import com.tencent.bkrepo.oci.util.OciLocationUtils +import com.tencent.bkrepo.oci.util.OciLocationUtils.buildBlobsFolderPath import com.tencent.bkrepo.oci.util.OciResponseUtils import com.tencent.bkrepo.oci.util.OciUtils import com.tencent.bkrepo.repository.api.MetadataClient @@ -91,17 +99,27 @@ import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import com.tencent.bkrepo.repository.api.PackageMetadataClient import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.repository.api.StorageCredentialsClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest import com.tencent.bkrepo.repository.pojo.packages.VersionListOption import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder import com.tencent.bkrepo.repository.pojo.search.PackageQueryBuilder +import com.tencent.devops.plugin.api.PluginManager +import com.tencent.devops.plugin.api.applyExtension import org.apache.commons.lang3.StringUtils import org.slf4j.Logger import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.where import org.springframework.stereotype.Service import java.nio.charset.Charset import java.time.Instant @@ -119,8 +137,10 @@ class OciOperationServiceImpl( private val storageManager: StorageManager, private val repositoryClient: RepositoryClient, private val ociProperties: OciProperties, - private val ociReplicationRecordDao: OciReplicationRecordDao -) : OciOperationService { + private val ociReplicationRecordDao: OciReplicationRecordDao, + private val storageCredentialsClient: StorageCredentialsClient, + private val pluginManager: PluginManager + ) : OciOperationService { /** * 检查package 对应的version是否存在 @@ -142,77 +162,6 @@ class OciOperationServiceImpl( ) } - /** - * 保存节点元数据 - */ - override fun saveMetaData( - projectId: String, - repoName: String, - fullPath: String, - metadata: MutableMap, - userId: String - ) { - val metadataSaveRequest = ObjectBuildUtils.buildMetadataSaveRequest( - projectId = projectId, - repoName = repoName, - fullPath = fullPath, - metadata = metadata, - userId = userId - ) - metadataClient.saveMetadata(metadataSaveRequest) - } - - /** - * 当mediaType为CHART_LAYER_MEDIA_TYPE,需要解析chart.yaml文件 - */ - override fun loadArtifactInput( - chartDigest: String?, - projectId: String, - repoName: String, - packageName: String, - version: String, - storageCredentials: StorageCredentials? - ): Map? { - if (chartDigest.isNullOrBlank()) return null - val blobDigest = OciDigest(chartDigest) - val fullPath = OciLocationUtils.buildDigestBlobsPath(packageName, blobDigest) - nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let { node -> - logger.info( - "Will read chart.yaml data from $fullPath with package $packageName " + - "and version $version under repo $projectId/$repoName" - ) - storageManager.loadArtifactInputStream(node, storageCredentials)?.let { - return try { - OciUtils.convertToMap(OciUtils.parseChartInputStream(it)) - } catch (e: Exception) { - logger.warn("Convert chart.yaml error: ${e.message}") - null - } - } - } - return null - } - - /** - * 需要将blob中相关metadata写进package version中 - */ - override fun updatePackageInfo( - ociArtifactInfo: OciArtifactInfo, - packageKey: String, - appVersion: String?, - description: String? - ) { - with(ociArtifactInfo) { - val packageUpdateRequest = ObjectBuildUtils.buildPackageUpdateRequest( - artifactInfo = this, - name = packageName, - appVersion = appVersion, - description = description, - packageKey = packageKey - ) - packageClient.updatePackage(packageUpdateRequest) - } - } /** * 删除package @@ -224,10 +173,10 @@ class OciOperationServiceImpl( val packageKey = PackageKeys.ofName(repoDetail.type.name.toLowerCase(), packageName) if (version.isNotBlank()) { packageClient.findVersionByName( - projectId, - repoName, - packageKey, - version + projectId = projectId, + repoName = repoName, + packageKey = packageKey, + version = version ).data?.let { removeVersion( artifactInfo = this, @@ -236,58 +185,26 @@ class OciOperationServiceImpl( packageKey = packageKey ) } ?: throw VersionNotFoundException(version) + if (!packageClient.listAllVersion(projectId, repoName, packageKey).data.isNullOrEmpty()) { + return + } + // 当没有版本时删除删除package目录 + deleteNode(projectId, repoName, "${StringPool.SLASH}$packageName", userId) } else { - packageClient.listAllVersion( + // 删除package目录 + deleteNode(projectId, repoName, "${StringPool.SLASH}$packageName", userId) + //删除package下所有版本 + packageClient.deletePackage( projectId, repoName, packageKey - ).data.orEmpty().forEach { - removeVersion( - artifactInfo = this, - version = it.name, - userId = userId, - packageKey = packageKey - ) - } - } - updatePackageExtension(artifactInfo, packageKey) - } - } - - /** - * 节点删除后,将package extension信息更新 - */ - private fun updatePackageExtension(artifactInfo: OciArtifactInfo, packageKey: String) { - with(artifactInfo) { - val version = packageClient.findPackageByKey(projectId, repoName, packageKey).data?.latest - try { - val chartDigest = findHelmChartYamlInfo(this, version) - val chartYaml = loadArtifactInput( - chartDigest = chartDigest, - projectId = projectId, - repoName = repoName, - packageName = packageName, - version = version!!, - storageCredentials = getRepositoryInfo(this).storageCredentials ) - // 针对helm chart包,将部分信息放入到package中 - chartYaml?.let { - val (appVersion, description) = getMetaDataFromChart(chartYaml) - updatePackageInfo( - ociArtifactInfo = artifactInfo, - appVersion = appVersion, - description = description, - packageKey = packageKey - ) - } - } catch (e: Exception) { - logger.warn("can not convert meta data") } } } /** - * 删除[version] 对应的node节点也会一起删除 + * 删除[version] 默认会删除对应的节点 */ private fun removeVersion( artifactInfo: OciArtifactInfo, @@ -296,78 +213,51 @@ class OciOperationServiceImpl( packageKey: String ) { with(artifactInfo) { - val nodeDetail = getBlobNodeDetail( - projectId = projectId, - repoName = repoName, - name = artifactInfo.packageName, - version = version - ) ?: return - // 删除manifest - deleteNode( - projectId = projectId, - repoName = repoName, - packageName = packageName, - path = nodeDetail.fullPath, + // 删除对应版本下所有关联的节点,包含manifest以及blobs + deleteVersionRelatedNodes( + artifactInfo = artifactInfo, + version = version, userId = userId ) packageClient.deleteVersion(projectId, repoName, packageKey, version) } } + /** - * 针对helm特殊处理,查找chart对应的digest,用于读取对应的chart.yaml信息 + * 删除对应版本下所有关联的节点,包含manifest以及blobs */ - private fun findHelmChartYamlInfo(artifactInfo: OciArtifactInfo, version: String? = null): String? { + private fun deleteVersionRelatedNodes( + artifactInfo: OciArtifactInfo, + version: String, + userId: String, + ) { with(artifactInfo) { - if (version.isNullOrBlank()) return null - val nodeDetail = getBlobNodeDetail( - projectId = projectId, - repoName = repoName, - name = artifactInfo.packageName, - version = version - ) ?: return null - val inputStream = storageService.load( - digest = nodeDetail.sha256!!, - range = Range.full(nodeDetail.size), - storageCredentials = getRepositoryInfo(artifactInfo).storageCredentials - ) ?: return null - try { - val manifest = OciUtils.streamToManifestV2(inputStream) - return (OciUtils.manifestIterator(manifest, CHART_LAYER_MEDIA_TYPE) ?: return null).digest - } catch (e: OciBadRequestException) { - logger.warn("Manifest convert error: ${e.message}") - } - return null + val manifestFolder = OciLocationUtils.buildManifestVersionFolderPath(packageName, version) + val blobsFolder = OciLocationUtils.blobVersionFolderLocation(version, packageName) + logger.info("Will delete blobsFolder [$blobsFolder] and manifestFolder $manifestFolder " + + "in package $packageName|$version in repo [$projectId/$repoName]") + // 删除manifestFolder + deleteNode(projectId, repoName, manifestFolder, userId) + // 删除blobs + deleteNode(projectId, repoName, blobsFolder, userId) } } + private fun deleteNode( projectId: String, repoName: String, - packageName: String, - userId: String, - digestStr: String? = null, - path: String? = null + fullPath: String, + userId: String ) { - val fullPath = path ?: digestStr?.let { - val nodeDetail = getBlobNodeDetail( - projectId = projectId, - repoName = repoName, - name = packageName, - digestStr = digestStr - ) - nodeDetail?.fullPath - } - fullPath?.let { - logger.info("Will delete node [$fullPath] with package $packageName in repo [$projectId/$repoName]") - val request = NodeDeleteRequest( - projectId = projectId, - repoName = repoName, - fullPath = fullPath, - operator = userId - ) - nodeClient.deleteNode(request) - } + val request = NodeDeleteRequest( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + operator = userId + ) + nodeClient.deleteNode(request) } /** @@ -398,7 +288,7 @@ class OciOperationServiceImpl( packageKey = packageKey, version = version ) - val nodeDetail = getBlobNodeDetail( + val nodeDetail = getImageNodeDetail( projectId = projectId, repoName = repoName, name = name, @@ -416,7 +306,7 @@ class OciOperationServiceImpl( * 获取node节点 * 查不到抛出OciFileNotFoundException异常 */ - private fun getBlobNodeDetail( + private fun getImageNodeDetail( projectId: String, repoName: String, name: String, @@ -476,14 +366,16 @@ class OciOperationServiceImpl( */ private fun buildNodeCreateRequest( ociArtifactInfo: OciArtifactInfo, - artifactFile: ArtifactFile + artifactFile: ArtifactFile, + metadata: List? = null ): NodeCreateRequest { return ObjectBuildUtils.buildNodeCreateRequest( projectId = ociArtifactInfo.projectId, repoName = ociArtifactInfo.repoName, artifactFile = artifactFile, - fullPath = ociArtifactInfo.getArtifactFullPath() + fullPath = ociArtifactInfo.getArtifactFullPath(), + metadata = metadata ) } @@ -498,42 +390,27 @@ class OciOperationServiceImpl( fileInfo: FileInfo?, proxyUrl: String? ): NodeDetail? { - val request = buildNodeCreateRequest(ociArtifactInfo, artifactFile) + // 用于新版本 blobs 路径区分, blob存储路径由 /{package}/blobs/转为/{package}/blobs/{version}/ + val metadata: MutableList = mutableListOf( + MetadataModel(key = BLOB_PATH_VERSION_KEY, value = BLOB_PATH_VERSION_VALUE, system = true) + ) + proxyUrl?.let { + metadata.add(MetadataModel(key = PROXY_URL, value = proxyUrl, system = true)) + } + val request = buildNodeCreateRequest(ociArtifactInfo, artifactFile, metadata) val nodeDetail = if (fileInfo != null) { val newNodeRequest = request.copy( size = fileInfo.size, md5 = fileInfo.md5, sha256 = fileInfo.sha256 ) - createNode(newNodeRequest, storageCredentials) - null + nodeClient.createNode(newNodeRequest).data } else { storageManager.storeArtifactFile(request, artifactFile, storageCredentials) } - proxyUrl?.let { - saveMetaData( - projectId = request.projectId, - repoName = request.repoName, - fullPath = request.fullPath, - metadata = mutableMapOf(PROXY_URL to proxyUrl), - userId = SecurityUtils.getUserId() - ) - } return nodeDetail } - /** - * 当使用追加上传时,文件已存储,只需存储节点信息 - */ - override fun createNode(request: NodeCreateRequest, storageCredentials: StorageCredentials?): NodeDetail { - try { - return nodeClient.createNode(request).data!! - } catch (exception: Exception) { - // 异常往上抛 - throw exception - } - } - /** * 更新整个blob相关信息,blob相关的mediatype,version等信息需要从manifest中获取 */ @@ -548,27 +425,20 @@ class OciOperationServiceImpl( "Will start to update oci info for ${ociArtifactInfo.getArtifactFullPath()} " + "in repo ${ociArtifactInfo.getRepoIdentify()}" ) - val manifestBytes = storageService.load( - nodeDetail.sha256.orEmpty(), - Range.full(nodeDetail.size), - storageCredentials - )!!.readText() - val schemaVersion = OciUtils.schemeVersion(manifestBytes) - // 将该版本对应的blob sha256放到manifest节点的元数据中 - var digestList: List? = null - val (mediaType, manifest) = if (schemaVersion.schemaVersion == 1) { - Pair(DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1, null) + // https://github.com/docker/docker-ce/blob/master/components/engine/distribution/push_v2.go + // docker 客户端上传manifest时先按照schema2的格式上传, + // 如失败则按照schema1格式上传,但是非docker客户端不兼容schema1版本manifest + val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) + ?: throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_SCHEMA1_NOT_SUPPORT) + // 更新manifest文件的metadata + val mediaType = if (manifest.mediaType.isNullOrEmpty()) { + HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE } else { - val manifest = OciUtils.stringToManifestV2(manifestBytes) - // 更新manifest文件的metadata - val mediaTypeV2 = if (manifest.mediaType.isNullOrEmpty()) { - HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE - } else { - manifest.mediaType - } - digestList = OciUtils.manifestIteratorDigest(manifest) - Pair(mediaTypeV2, manifest) + manifest.mediaType } + val digestList = OciUtils.manifestIteratorDigest(manifest) + + // 更新manifest节点元数据 updateNodeMetaData( projectId = ociArtifactInfo.projectId, repoName = ociArtifactInfo.repoName, @@ -578,25 +448,34 @@ class OciOperationServiceImpl( digestList = digestList, sourceType = sourceType ) - // 同步blob相关metadata - if (ociArtifactInfo.packageName.isNotEmpty()) { - if (schemaVersion.schemaVersion == 1) { - syncBlobInfoV1( - ociArtifactInfo = ociArtifactInfo, - manifestDigest = digest, - manifestPath = nodeDetail.fullPath, - sourceType = sourceType - ) - } else { - syncBlobInfo( - ociArtifactInfo = ociArtifactInfo, - manifest = manifest!!, - manifestDigest = digest, - storageCredentials = storageCredentials, - manifestPath = nodeDetail.fullPath, - sourceType = sourceType - ) - } + + + if (ociArtifactInfo.packageName.isEmpty()) return + // 处理manifest中的blob数据 + syncBlobInfo( + ociArtifactInfo = ociArtifactInfo, + manifest = manifest, + nodeDetail = nodeDetail, + sourceType = sourceType + ) + } + + + private fun loadManifest( + sha256: String, + size: Long, + storageCredentials: StorageCredentials? + ): ManifestSchema2? { + return try { + val manifestBytes = storageService.load( + sha256, + Range.full(size), + storageCredentials + )!!.readText() + + OciUtils.stringToManifestV2(manifestBytes) + } catch (e: Exception) { + null } } @@ -607,34 +486,16 @@ class OciOperationServiceImpl( with(ociArtifactInfo) { val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data ?: return false val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, manifestPath).data ?: return false - val manifestBytes = storageService.load( - nodeDetail.sha256!!, - Range.full(nodeDetail.size), - repositoryDetail.storageCredentials - )!!.readText() - val manifest = OciUtils.stringToManifestV2(manifestBytes) - val descriptorList = OciUtils.manifestIterator(manifest) - // 用于判断是否所有blob都以存在 - var existFlag: Boolean - var size: Long = 0 - - descriptorList.forEach { - size += it.size - existFlag = doSyncBlob(it, ociArtifactInfo, repositoryDetail.storageCredentials) - // 如果当前镜像下的blob没有全部存储在制品库,则不生成版本,由定时任务去生成 - if (!existFlag) return false - } - // 根据flag生成package信息以及package version信息 - doPackageOperations( - manifestPath = manifestPath, + val manifest = loadManifest( + nodeDetail.sha256!!, nodeDetail.size, repositoryDetail.storageCredentials + ) ?: return false + return syncBlobInfo( ociArtifactInfo = ociArtifactInfo, - manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), - size = size, - chartYaml = null, + manifest = manifest, + nodeDetail = nodeDetail, sourceType = ArtifactChannel.REPLICATION, userId = SYSTEM_USER ) - return true } } @@ -647,7 +508,6 @@ class OciOperationServiceImpl( version: String? = null, fullPath: String, mediaType: String, - chartYaml: Map? = null, digestList: List? = null, sourceType: ArtifactChannel? = null ) { @@ -655,111 +515,127 @@ class OciOperationServiceImpl( val metadata = ObjectBuildUtils.buildMetadata( mediaType = mediaType, version = version, - yamlData = chartYaml, digestList = digestList, sourceType = sourceType ) - saveMetaData( + + updateNodeMetaData( projectId = projectId, repoName = repoName, fullPath = fullPath, - metadata = metadata, - userId = SecurityUtils.getUserId() + metadata = metadata ) } - /** - * 同步fsLayers层的数据 - */ - private fun syncBlobInfoV1( - ociArtifactInfo: OciManifestArtifactInfo, - manifestDigest: OciDigest, - manifestPath: String, - sourceType: ArtifactChannel? = null + + private fun updateNodeMetaData( + projectId: String, + repoName: String, + fullPath: String, + metadata: Map ) { - logger.info( - "Will start to sync fsLayers' blob info from manifest ${ociArtifactInfo.getArtifactFullPath()} " + - "to blobs in repo ${ociArtifactInfo.getRepoIdentify()}." - ) - // 根据flag生成package信息以及packageversion信息 - doPackageOperations( - manifestPath = manifestPath, - ociArtifactInfo = ociArtifactInfo, - manifestDigest = manifestDigest, - size = 0, - sourceType = sourceType + val metadataSaveRequest = ObjectBuildUtils.buildMetadataSaveRequest( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + metadata = metadata, + userId = SecurityUtils.getUserId() ) + try{ + metadataClient.saveMetadata(metadataSaveRequest) + } catch (ignore: Exception) { + // 并发情况下会出现节点找不到问题 + } } + + /** * 同步blob层的数据和config里面的数据 */ private fun syncBlobInfo( ociArtifactInfo: OciManifestArtifactInfo, + nodeDetail: NodeDetail, + sourceType: ArtifactChannel? = null, manifest: ManifestSchema2, - manifestDigest: OciDigest, - storageCredentials: StorageCredentials?, - manifestPath: String, - sourceType: ArtifactChannel? = null - ) { + userId: String = SecurityUtils.getUserId() + ): Boolean { logger.info( "Will start to sync blobs and config info from manifest ${ociArtifactInfo.getArtifactFullPath()} " + - "to blobs in repo ${ociArtifactInfo.getRepoIdentify()}." + "to blobs in repo ${ociArtifactInfo.getRepoIdentify()}." + ) + // existFlag 判断manifest里的所有blob是否都已经创建节点 + // size 整个镜像blob汇总的大小 + val (existFlag, size) = manifestHandler( + manifest, ociArtifactInfo, userId ) - val descriptorList = OciUtils.manifestIterator(manifest) - // 用于判断是否所有blob都以存在 - var existFlag = true - var chartYaml: Map? = null - // 统计所有manifest中的文件size作为整个package version的size - var size: Long = 0 - // 同步layer以及config层blob信息 - descriptorList.forEach { - size += it.size - chartYaml = when (it.mediaType) { - CHART_LAYER_MEDIA_TYPE -> { - // 针对helm chart,需要将chart.yaml中相关信息存入对应节点中 - loadArtifactInput( - chartDigest = it.digest, - projectId = ociArtifactInfo.projectId, - repoName = ociArtifactInfo.repoName, - packageName = ociArtifactInfo.packageName, - version = ociArtifactInfo.reference, - storageCredentials = storageCredentials - ) - } - else -> null - } - existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, storageCredentials) - } // 如果当前镜像下的blob没有全部存储在制品库,则不生成版本,由定时任务去生成 if (existFlag) { + // 第三方同步的索引更新等所有文件全部上传完成后才去进行 // 根据flag生成package信息以及package version信息 doPackageOperations( - manifestPath = manifestPath, + manifestPath = nodeDetail.fullPath, ociArtifactInfo = ociArtifactInfo, - manifestDigest = manifestDigest, + manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), size = size, - chartYaml = chartYaml, - sourceType = sourceType + sourceType = sourceType, + userId = userId ) + return true } else { - ociReplicationRecordDao.save(TOciReplicationRecord( - projectId = ociArtifactInfo.projectId, - repoName = ociArtifactInfo.repoName, - packageName = ociArtifactInfo.packageName, - packageVersion = ociArtifactInfo.reference, - manifestPath = manifestPath - )) + val query = Query.query( + where(TOciReplicationRecord::projectId).isEqualTo(ociArtifactInfo.projectId) + .and(TOciReplicationRecord::repoName).isEqualTo(ociArtifactInfo.repoName) + .and(TOciReplicationRecord::packageName).isEqualTo(ociArtifactInfo.packageName) + .and(TOciReplicationRecord::packageVersion).isEqualTo(ociArtifactInfo.reference) + ) + val update = Update().setOnInsert(TOciReplicationRecord::manifestPath.name, nodeDetail.fullPath) + ociReplicationRecordDao.upsert(query, update) + return false + } + } + + + /** + * 针对v2版本manifest文件做特殊处理 + */ + private fun manifestHandler( + manifest: ManifestSchema2, + ociArtifactInfo: OciManifestArtifactInfo, + userId: String = SecurityUtils.getUserId() + ): Pair { + // 用于判断是否所有blob都以存在 + var existFlag = true + // 统计所有mainfest中的文件size作为整个package version的size + var size: Long = 0 + val descriptorList = OciUtils.manifestIterator(manifest) + + // 当同一版本覆盖上传时,先删除当前版本对应的blobs目录,然后再创建 + val blobsFolder = OciLocationUtils.blobVersionFolderLocation( + ociArtifactInfo.reference, ociArtifactInfo.packageName + ) + deleteNode(ociArtifactInfo.projectId, ociArtifactInfo.repoName, blobsFolder, userId) + + // 同步layer以及config层blob信息 + descriptorList.forEach { + size += it.size + existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, userId) + if (!existFlag) { + // 第三方同步场景下,如果当前镜像下的blob没有全部存储在制品库,则不生成版本,由定时任务去生成 + return Pair(false, 0) + } } + return Pair(existFlag, size) } + /** * 更新blobs的信息 */ private fun doSyncBlob( descriptor: Descriptor, ociArtifactInfo: OciManifestArtifactInfo, - storageCredentials: StorageCredentials? + userId: String = SecurityUtils.getUserId() ): Boolean { with(ociArtifactInfo) { logger.info( @@ -775,7 +651,7 @@ class OciOperationServiceImpl( fullPath = fullPath, descriptor = descriptor, ociArtifactInfo = this, - storageCredentials = storageCredentials + userId = userId ) } } @@ -787,23 +663,36 @@ class OciOperationServiceImpl( fullPath: String, descriptor: Descriptor, ociArtifactInfo: OciManifestArtifactInfo, - storageCredentials: StorageCredentials? + userId: String = SecurityUtils.getUserId() ): Boolean { with(ociArtifactInfo) { - val blobExist = nodeClient.checkExist(projectId, repoName, fullPath).data!! - // blob节点不存在,但是在同一仓库下存在digest文件 - if (!blobExist) { - val (existFullPath, md5) = getNodeByDigest(projectId, repoName, descriptor.digest) - if (existFullPath == null) return false + // 并发情况下,版本目录下可能存在着非该版本的blob + // 覆盖上传时会先删除原有目录,并发情况下可能导致blobs节点不存在 + val nodeProperty = getNodeByDigest(projectId, repoName, descriptor.digest) ?: run { + nodeClient.getDeletedNodeDetailBySha256( + projectId, repoName, descriptor.sha256).data?.let { + NodeProperty(StringPool.EMPTY, it.md5, it.size) + } ?: return false + } + val newPath = OciLocationUtils.blobVersionPathLocation(reference, packageName, descriptor.filename) + if (newPath != nodeProperty.fullPath) { + // 创建新路径节点 /packageName/blobs/version/xxx val nodeCreateRequest = ObjectBuildUtils.buildNodeCreateRequest( projectId = projectId, repoName = repoName, - size = descriptor.size, + size = nodeProperty.size!!, sha256 = descriptor.sha256, - fullPath = fullPath, - md5 = md5 ?: StringPool.UNKNOWN + fullPath = newPath, + md5 = nodeProperty.md5 ?: StringPool.UNKNOWN, + userId = userId ) - createNode(nodeCreateRequest, storageCredentials) + nodeClient.createNode(nodeCreateRequest) + } + val metadataMap = metadataClient.listMetadata(projectId, repoName, fullPath).data + if (metadataMap?.get(BLOB_PATH_VERSION_KEY) != null) { + // 只有当新建的blob路径节点才去删除,历史的由定时任务去刷新然后删除 + // 删除临时存储路径节点 /packageName/blobs/xxx + deleteNode(projectId, repoName, fullPath, userId) } return true } @@ -817,7 +706,6 @@ class OciOperationServiceImpl( ociArtifactInfo: OciManifestArtifactInfo, manifestDigest: OciDigest, size: Long, - chartYaml: Map? = null, sourceType: ArtifactChannel? = null, userId: String = SecurityUtils.getUserId() ) { @@ -828,7 +716,6 @@ class OciOperationServiceImpl( val packageKey = PackageKeys.ofName(repoType.toLowerCase(), packageName) val metadata = mutableMapOf(MANIFEST_DIGEST to manifestDigest.toString()) .apply { - chartYaml?.let { this.putAll(chartYaml) } sourceType?.let { this[SOURCE_TYPE] = sourceType } } val request = ObjectBuildUtils.buildPackageVersionCreateRequest( @@ -848,17 +735,6 @@ class OciOperationServiceImpl( version = ociArtifactInfo.reference, metadata = metadata ) - - // 针对helm chart包,将部分信息放入到package中 - chartYaml?.let { - val (appVersion, description) = getMetaDataFromChart(chartYaml) - updatePackageInfo( - ociArtifactInfo = ociArtifactInfo, - appVersion = appVersion, - description = description, - packageKey = packageKey - ) - } } } @@ -880,36 +756,28 @@ class OciOperationServiceImpl( metadata = metadata, userId = SecurityUtils.getUserId() ) - packageMetadataClient.saveMetadata(metadataSaveRequest) - } - - /** - * 针对helm chart包,将部分信息放入到package中 - */ - private fun getMetaDataFromChart(chartYaml: Map? = null): Pair { - var appVersion: String? = null - var description: String? = null - chartYaml?.let { - appVersion = it[APP_VERSION] as String? - description = it[DESCRIPTION] as String? + try { + packageMetadataClient.saveMetadata(metadataSaveRequest) + } catch (ignore: Exception) { } - return Pair(appVersion, description) } + /** * 获取对应存储节点路径 * 特殊:manifest文件按tag存储, 但是查询时存在tag/digest */ override fun getNodeFullPath(artifactInfo: OciArtifactInfo): String? { - if (artifactInfo is OciManifestArtifactInfo) { - // 根据类型解析实际存储路径,manifest获取路径有tag/digest - if (artifactInfo.isValidDigest) { - return getNodeByDigest( - projectId = artifactInfo.projectId, - repoName = artifactInfo.repoName, - digestStr = artifactInfo.reference - ).fullPath - } + if (artifactInfo is OciManifestArtifactInfo && artifactInfo.isValidDigest) { + // 根据类型解析实际存储路径,manifest获取路径有tag/digest, + // 当为digest 时,查当前仓库下,在package目录下,且sha256为digest的节点 + val path = OciLocationUtils.buildManifestFolderPath(artifactInfo.packageName) + return getNodeByDigest( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + digestStr = artifactInfo.reference, + path = path + )?.fullPath } return artifactInfo.getArtifactFullPath() } @@ -920,23 +788,28 @@ class OciOperationServiceImpl( override fun getNodeByDigest( projectId: String, repoName: String, - digestStr: String - ): NodeProperty { + digestStr: String, + path: String? + ): NodeProperty? { val ociDigest = OciDigest(digestStr) val queryModel = NodeQueryBuilder() .select(NODE_FULL_PATH, MD5, OCI_NODE_SIZE) .projectId(projectId) .repoName(repoName) .sha256(ociDigest.getDigestHex()) - .sortByAsc(NODE_FULL_PATH) - val result = nodeClient.search(queryModel.build()).data ?: run { + .sortByAsc(NODE_FULL_PATH).apply { + path?.let { + this.path(path, OperationType.PREFIX) + } + } + val result = nodeClient.search(queryModel.build()).data + if (result == null || result.records.isEmpty()) { logger.warn( "Could not find $digestStr " + "in repo $projectId|$repoName" ) - return NodeProperty() + return null } - if (result.records.isEmpty()) return NodeProperty() return NodeProperty( fullPath = result.records[0][NODE_FULL_PATH] as String, md5 = result.records[0][MD5] as String?, @@ -951,25 +824,29 @@ class OciOperationServiceImpl( * 2 docker-local/nginx/_uploads/ 临时存储上传的blobs,待manifest文件上传成功后移到到对应版本下,如docker-local/nginx/latest */ override fun getDockerNode(artifactInfo: OciArtifactInfo): String? { - if (artifactInfo is OciManifestArtifactInfo) { - // 根据类型解析实际存储路径,manifest获取路径有tag/digest - if (artifactInfo.isValidDigest) + return when (artifactInfo) { + is OciManifestArtifactInfo -> { + // 根据类型解析实际存储路径,manifest获取路径有tag/digest + if (artifactInfo.isValidDigest){ + return getNodeByDigest( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + digestStr = artifactInfo.reference, + path = "/${artifactInfo.packageName}/" + )?.fullPath + } + return "/${artifactInfo.packageName}/${artifactInfo.reference}/manifest.json" + } + is OciBlobArtifactInfo -> { + val digestStr = artifactInfo.digest ?: StringPool.EMPTY return getNodeByDigest( projectId = artifactInfo.projectId, repoName = artifactInfo.repoName, - digestStr = artifactInfo.reference - ).fullPath - return "/${artifactInfo.packageName}/${artifactInfo.reference}/manifest.json" - } - if (artifactInfo is OciBlobArtifactInfo) { - val digestStr = artifactInfo.digest ?: StringPool.EMPTY - return getNodeByDigest( - projectId = artifactInfo.projectId, - repoName = artifactInfo.repoName, - digestStr = digestStr - ).fullPath + digestStr = digestStr + )?.fullPath + } + else -> null } - return null } override fun getReturnDomain(request: HttpServletRequest): String { @@ -1093,6 +970,144 @@ class OciOperationServiceImpl( return OciTagResult(result.totalRecords, data) } + override fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String) { + val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data + ?: throw RepoNotFoundException("$projectId|$repoName") + buildImagePackagePullContext(projectId, repoName, repositoryDetail.configuration).forEach { + pluginManager.applyExtension { + queryAndCreateDockerPackageInfo(it) + } + } + } + + override fun deleteBlobsFolderAfterRefreshed( + projectId: String, repoName: String, pName: String, userId: String + ) { + repositoryClient.getRepoInfo(projectId, repoName).data + ?: throw RepoNotFoundException("$projectId|$repoName") + val blobsFolderPath = buildBlobsFolderPath(pName) + val fullPaths = nodeClient.listNode( + projectId, repoName, blobsFolderPath, includeFolder = false, deep = false + ).data?.map { it.fullPath } + if (fullPaths.isNullOrEmpty()) return + logger.info("Blobs of package $pName in folder $blobsFolderPath will be deleted in $projectId|$repoName") + val request = NodesDeleteRequest( + projectId = projectId, + repoName = repoName, + fullPaths = fullPaths, + operator = userId + ) + nodeClient.deleteNodes(request) + } + + /** + * 调整blob目录 + * 从/packageName/blobs/xxx 到/packageName/blobs/version/xxx + */ + override fun refreshBlobNode( + projectId: String, + repoName: String, + pName: String, + pVersion: String, + userId: String + ): Boolean { + val repoInfo = repositoryClient.getRepoInfo(projectId, repoName).data + ?: throw RepoNotFoundException("$projectId|$repoName") + val packageName = PackageKeys.resolveName(repoInfo.type.name.toLowerCase(), pName) + val manifestPath = OciLocationUtils.buildManifestPath(packageName, pVersion) + logger.info("Manifest $manifestPath will be refreshed") + val ociArtifactInfo = OciManifestArtifactInfo( + projectId = repoInfo.projectId, + repoName = repoInfo.name, + packageName = packageName, + version = pVersion, + reference = pVersion, + isValidDigest = false + ) + val manifestNode = nodeClient.getNodeDetail( + repoInfo.projectId, repoInfo.name, manifestPath + ).data ?: run { + val oldDockerFullPath = getDockerNode(ociArtifactInfo) ?: return false + nodeClient.getNodeDetail( + repoInfo.projectId, repoInfo.name, oldDockerFullPath + ).data ?: return false + } + val refreshedMetadat = manifestNode.nodeMetadata.firstOrNull { it.key == BLOB_PATH_REFRESHED_KEY} + if (refreshedMetadat != null) { + logger.info("$manifestPath has been refreshed, ignore it") + return true + } + val storageCredentials = repoInfo.storageCredentialsKey?.let { storageCredentialsClient.findByKey(it).data } + val manifest = loadManifest( + manifestNode.sha256!!, manifestNode.size, storageCredentials + ) ?: run { + logger.warn("The content of manifest.json ${manifestNode.fullPath} is null, check the mediaType.") + return false + } + var refreshStatus = true + OciUtils.manifestIterator(manifest).forEach { + refreshStatus = refreshStatus && doSyncBlob(it, ociArtifactInfo, userId) + } + + if (refreshStatus) { + updateNodeMetaData( + projectId = repoInfo.projectId, + repoName = repoInfo.name, + fullPath = manifestPath, + metadata = mapOf(BLOB_PATH_REFRESHED_KEY to true) + ) + } + logger.info("The status of path $manifestPath refreshed is $refreshStatus") + return refreshStatus + } + + + + + + + private fun buildImagePackagePullContext( + projectId: String, + repoName: String, + config: RepositoryConfiguration + ): List { + val result = mutableListOf() + when (config) { + is RemoteConfiguration -> { + try { + val remoteUrl = UrlFormatter.addProtocol(config.url) + result.add(ImagePackagePullContext( + projectId = projectId, + repoName = repoName, + remoteUrl = remoteUrl, + userName = config.credentials.username, + password = config.credentials.password + )) + } catch (e: Exception) { + logger.warn("illegal remote url ${config.url} for repo $projectId|$repoName") + } + } + is CompositeConfiguration -> { + config.proxy.channelList.forEach { + try { + val remoteUrl = UrlFormatter.addProtocol(it.url) + result.add(ImagePackagePullContext( + projectId = projectId, + repoName = repoName, + remoteUrl = remoteUrl, + userName = it.username, + password = it.password + )) + } catch (e: Exception) { + logger.warn("illegal proxy url ${it.url} for repo $projectId|$repoName") + } + } + } + else -> throw UnsupportedOperationException() + } + return result + } + private fun buildString(stageTag: List): String { if (stageTag.isEmpty()) return StringPool.EMPTY return StringUtils.join(stageTag.toTypedArray()).toString() diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt index e5a9fd2bd9..c7a3c534d8 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt @@ -33,6 +33,7 @@ import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.security.util.SecurityUtils +import com.tencent.bkrepo.oci.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.oci.constant.DIGEST_LIST import com.tencent.bkrepo.oci.constant.IMAGE_VERSION import com.tencent.bkrepo.oci.constant.MEDIA_TYPE @@ -76,7 +77,8 @@ object ObjectBuildUtils { fullPath: String, sha256: String, md5: String, - metadata: List? = null + metadata: List? = null, + userId: String = SecurityUtils.getUserId() ): NodeCreateRequest { return NodeCreateRequest( projectId = projectId, @@ -86,7 +88,7 @@ object ObjectBuildUtils { size = size, sha256 = sha256, md5 = md5, - operator = SecurityUtils.getUserId(), + operator = userId, overwrite = true, nodeMetadata = metadata ) @@ -95,19 +97,16 @@ object ObjectBuildUtils { fun buildMetadata( mediaType: String, version: String?, - yamlData: Map? = null, digestList: List? = null, sourceType: ArtifactChannel? = null ): MutableMap { return mutableMapOf( - MEDIA_TYPE to mediaType + MEDIA_TYPE to mediaType, + BLOB_PATH_REFRESHED_KEY to true ).apply { version?.let { this.put(IMAGE_VERSION, version) } digestList?.let { this.put(DIGEST_LIST, digestList) } sourceType?.let { this.put(SOURCE_TYPE, sourceType) } - yamlData?.let { - this.putAll(yamlData) - } } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt index 674c1888c8..6cca3c70a4 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt @@ -49,6 +49,7 @@ import com.tencent.bkrepo.oci.constant.HTTP_PROTOCOL_HTTP import com.tencent.bkrepo.oci.constant.HTTP_PROTOCOL_HTTPS import com.tencent.bkrepo.oci.constant.OCI_API_PREFIX import com.tencent.bkrepo.oci.pojo.digest.OciDigest +import com.tencent.bkrepo.oci.pojo.response.ResponseProperty import org.springframework.http.HttpHeaders import java.io.UnsupportedEncodingException import java.net.URI @@ -99,58 +100,11 @@ object OciResponseUtils { ) } - fun buildUploadResponse(domain: String, digest: OciDigest, locationStr: String, response: HttpServletResponse) { + fun buildUploadResponse(domain: String, responseProperty: ResponseProperty, response: HttpServletResponse) { uploadResponse( domain = domain, response = response, - status = HttpStatus.CREATED, - locationStr = locationStr, - digest = digest.toString(), - contentLength = 0 - ) - } - - fun buildBlobUploadUUIDResponse(domain: String, uuid: String, locationStr: String, response: HttpServletResponse) { - uploadResponse( - domain = domain, - response = response, - status = HttpStatus.ACCEPTED, - locationStr = locationStr, - uuid = uuid, - contentLength = 0 - ) - } - - fun buildBlobUploadPatchResponse( - domain: String, - uuid: String, - locationStr: String, - status: HttpStatus = HttpStatus.ACCEPTED, - response: HttpServletResponse, - range: Long - ) { - uploadResponse( - domain = domain, - response = response, - status = status, - locationStr = locationStr, - uuid = uuid, - contentLength = 0, - range = range - ) - } - - fun buildBlobMountResponse( - domain: String, - locationStr: String, - status: HttpStatus = HttpStatus.ACCEPTED, - response: HttpServletResponse - ) { - uploadResponse( - domain = domain, - response = response, - status = status, - locationStr = locationStr + responseProperty = responseProperty ) } @@ -175,29 +129,26 @@ object OciResponseUtils { private fun uploadResponse( domain: String, response: HttpServletResponse = HttpContextHolder.getResponse(), - status: HttpStatus, - locationStr: String, - digest: String? = null, - uuid: String? = null, - range: Long? = null, - contentLength: Int? = null + responseProperty: ResponseProperty ) { - val location = getResponseLocationURI(locationStr, domain) - response.status = status.value - response.addHeader(DOCKER_HEADER_API_VERSION, DOCKER_API_VERSION) - digest?.let { - response.addHeader(DOCKER_CONTENT_DIGEST, digest) - } - response.addHeader(HttpHeaders.LOCATION, location) - uuid?.let { - response.addHeader(BLOB_UPLOAD_SESSION_ID, uuid) - response.addHeader(DOCKER_UPLOAD_UUID, uuid) - } - contentLength?.let { - response.addHeader(CONTENT_LENGTH, contentLength.toString()) - } - range?.let { - response.addHeader(RANGE, "0-${range - 1}") + with(responseProperty) { + val location = getResponseLocationURI(location!!, domain) + response.status = status!!.value + response.addHeader(DOCKER_HEADER_API_VERSION, DOCKER_API_VERSION) + digest?.let { + response.addHeader(DOCKER_CONTENT_DIGEST, digest.toString()) + } + response.addHeader(HttpHeaders.LOCATION, location) + uuid?.let { + response.addHeader(BLOB_UPLOAD_SESSION_ID, uuid) + response.addHeader(DOCKER_UPLOAD_UUID, uuid) + } + contentLength?.let { + response.addHeader(CONTENT_LENGTH, contentLength.toString()) + } + range?.let { + response.addHeader(RANGE, "0-${range!! - 1}") + } } } diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties index eb6c359888..1df4423048 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties @@ -36,4 +36,7 @@ oci.repo.not.found=Oci repository [{0}] not found! oci.delete.rules=Delete the blob [{0}] identified by digest! oci.version.not.found=The version [{0}] of artifact not found in repo [{1}] ! oci.manifest.invalid=Manifest invalid! -oci.digest.invalid=Digest invalid [{0}]! \ No newline at end of file +oci.digest.invalid=Digest invalid [{0}]! +oci.manifest.schema1.not.support=Schema1 of manifest not supported! +oci.remote.configuration.error= Remote configuration error, error is [{0}]! +oci.remote.credentials.invalid= Remote credentials invalid, error is [{0}]! \ No newline at end of file diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties index ba00b5bcdb..9e5117b24c 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties @@ -36,4 +36,7 @@ oci.repo.not.found=Oci仓库 [{0}] 不存在! oci.delete.rules=制品 [{0}] 只能使用digest删除! oci.version.not.found=制品版本 [{0}] 在仓库 [{1}] 中不存在! oci.manifest.invalid=请检查Manifest内容是否符合规范! -oci.digest.invalid=上传内容Digest与提供的digest不一致 [{0}]! \ No newline at end of file +oci.digest.invalid=上传内容Digest与提供的digest不一致 [{0}]! +oci.manifest.schema1.not.support=Manifest文件不支持Schema1版本! +oci.remote.configuration.error= 请检查远程仓库配置信息, 错误:[{0}]! +oci.remote.credentials.invalid= 代理鉴权信息有误: [{0}],请确认! \ No newline at end of file diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties index 59782463fa..c9599531b3 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties @@ -36,4 +36,7 @@ oci.repo.not.found=Oci倉庫 [{0}] 不存在! oci.delete.rules=製品 [{0}] 只能digest刪除! oci.version.not.found=製品版本 [{0}] 在倉庫 [{1}] 中不存在! oci.manifest.invalid=請檢查Manifest內容是否符合規範! -oci.digest.invalid=上傳內容Digest與提供的digest不一致 [{0}]! \ No newline at end of file +oci.digest.invalid=上傳內容Digest與提供的digest不一致 [{0}]! +oci.manifest.schema1.not.support=Manifest文件不支持Schema1版本! +oci.remote.configuration.error= 請檢查遠程倉庫配置信息, 錯誤:[{0}]! +oci.remote.credentials.invalid= 代理鑒權信息有誤: [{0}] ,請確認! \ No newline at end of file diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt index f98f68e889..cb08bd2168 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt +++ b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt @@ -29,9 +29,6 @@ package com.tencent.bkrepo.replication.constant const val DEFAULT_VERSION = "1.0.0" -const val BEARER_REALM = "Bearer realm" -const val SERVICE = "service" -const val SCOPE = "scope" const val REPOSITORY = "repository" const val URL = "url" @@ -43,6 +40,8 @@ const val DOCKER_MANIFEST_JSON_FULL_PATH = "/%s/%s/manifest.json" const val DOCKER_LAYER_FULL_PATH = "/%s/%s/%s" const val OCI_MANIFEST_JSON_FULL_PATH = "/%s/manifest/%s/manifest.json" const val OCI_LAYER_FULL_PATH = "/%s/blobs/%s" +const val OCI_LAYER_FULL_PATH_V1 = "/%s/blobs/%s/%s" +const val BLOB_PATH_REFRESHED_KEY = "blobPathRefreshed" const val NODE_FULL_PATH = "fullPath" const val SIZE = "size" const val REPOSITORY_INFO = "repo" diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt index dd029ce030..ee395dce11 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.constant.CHUNKED_UPLOAD import com.tencent.bkrepo.replication.constant.REPOSITORY_INFO @@ -44,7 +45,6 @@ import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import com.tencent.bkrepo.replication.pojo.request.ReplicaType import com.tencent.bkrepo.replication.replica.base.context.FilePushContext import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.util.HttpUtils import com.tencent.bkrepo.replication.util.StreamRequestBody import okhttp3.Headers import okhttp3.MediaType.Companion.toMediaTypeOrNull @@ -352,7 +352,7 @@ abstract class ArtifactReplicationHandler( } catch (e: Exception) { val baseUrl = URL(url) val host = URL(baseUrl.protocol, baseUrl.host, StringPool.EMPTY).toString() - HttpUtils.buildUrl(host, location.removePrefix("/")) + UrlFormatter.buildUrl(host, location.removePrefix("/")) } } } @@ -368,7 +368,7 @@ abstract class ArtifactReplicationHandler( ): String { val baseUrl = URL(url) val suffixUrl = URL(baseUrl, baseUrl.path).toString() - return HttpUtils.buildUrl(suffixUrl, path, params) + return UrlFormatter.buildUrl(suffixUrl, path, params) } open fun buildRequestTag( diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt index 9b96f26361..8dec006a3b 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt @@ -28,13 +28,13 @@ package com.tencent.bkrepo.replication.replica.base.handler import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.constant.OCI_BLOBS_UPLOAD_FIRST_STEP_URL import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.remote.DefaultHandlerResult import com.tencent.bkrepo.replication.replica.base.context.FilePushContext import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.util.HttpUtils import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.net.URL @@ -105,7 +105,7 @@ class RemoteClusterArtifactReplicationHandler( ): String { val baseUrl = URL(url) val suffixUrl = URL(baseUrl, "/v2" + baseUrl.path).toString() - return HttpUtils.buildUrl(suffixUrl, path, params) + return UrlFormatter.buildUrl(suffixUrl, path, params) } companion object { private val logger = LoggerFactory.getLogger(RemoteClusterArtifactReplicationHandler::class.java) diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt index 5df18e0027..bfb0450916 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt @@ -4,13 +4,16 @@ import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.replication.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.replication.constant.DOCKER_LAYER_FULL_PATH import com.tencent.bkrepo.replication.constant.DOCKER_MANIFEST_JSON_FULL_PATH import com.tencent.bkrepo.replication.constant.OCI_LAYER_FULL_PATH +import com.tencent.bkrepo.replication.constant.OCI_LAYER_FULL_PATH_V1 import com.tencent.bkrepo.replication.constant.OCI_MANIFEST_JSON_FULL_PATH import com.tencent.bkrepo.replication.util.ManifestParser import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import org.springframework.stereotype.Component @@ -61,15 +64,40 @@ class DockerPackageNodeMapper( throw ArtifactNotFoundException("Could not read manifest.json, $e") } manifestInfo!!.descriptors?.forEach { - val replace = it.replace(":", "__") - if (isOci) { - result.add(OCI_LAYER_FULL_PATH.format(name, replace)) - } else { - result.add(DOCKER_LAYER_FULL_PATH.format(name, version, replace)) - } + result.add(buildBlobPath( + descriptor = it, + packageName = name, + version = version, + isOci = isOci, + nodeDetail = nodeDetail + )) } result.add(manifestFullPath) return result } } + + /** + * 通过包名、版本、sha256拼接出blob路径 + */ + private fun buildBlobPath( + descriptor: String, + packageName: String, + version: String, + isOci: Boolean, + nodeDetail: NodeDetail + ): String { + val replace = descriptor.replace(":", "__") + return if (isOci) { + // 镜像blob路径格式有调整,从/package/blobs/下调至//package/blobs/version/ + val refreshedMetadata = nodeDetail.nodeMetadata.firstOrNull { it.key == BLOB_PATH_REFRESHED_KEY} + if (refreshedMetadata != null) { + OCI_LAYER_FULL_PATH_V1.format(packageName, version, replace) + } else { + OCI_LAYER_FULL_PATH.format(packageName, replace) + } + } else { + DOCKER_LAYER_FULL_PATH.format(packageName, version, replace) + } + } } diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt index 3639472d13..2e57791dc0 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt @@ -32,6 +32,7 @@ import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.BasicAuthUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.common.service.cluster.ClusterInfo import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager @@ -40,7 +41,6 @@ import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext import com.tencent.bkrepo.replication.replica.base.handler.DefaultHandler import com.tencent.bkrepo.replication.replica.base.impl.remote.base.PushClient -import com.tencent.bkrepo.replication.util.HttpUtils import com.tencent.bkrepo.repository.pojo.node.NodeDetail import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.MultipartBody @@ -258,7 +258,7 @@ class HelmArtifactPushClient( ): String { val baseUrl = URL(url) val v2Url = URL(baseUrl, baseUrl.path) - return HttpUtils.buildUrl(v2Url.toString(), path, params) + return UrlFormatter.buildUrl(v2Url.toString(), path, params) } private fun buildAuthRequestProperties(clusterInfo: ClusterInfo): RequestProperty { diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt index 2a7ca766f8..989ac50cf1 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt @@ -31,14 +31,11 @@ import com.tencent.bkrepo.common.api.constant.BEARER_AUTH_PREFIX import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.StringPool -import com.tencent.bkrepo.common.api.constant.StringPool.QUESTION +import com.tencent.bkrepo.common.api.util.AuthenticationUtil.buildAuthenticationUrl +import com.tencent.bkrepo.common.api.util.AuthenticationUtil.parseWWWAuthenticateHeader import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.toJsonString -import com.tencent.bkrepo.replication.constant.BEARER_REALM -import com.tencent.bkrepo.replication.constant.SCOPE -import com.tencent.bkrepo.replication.constant.SERVICE import com.tencent.bkrepo.replication.pojo.docker.OciResponse -import com.tencent.bkrepo.replication.pojo.remote.AuthenticationProperty import com.tencent.bkrepo.replication.pojo.remote.BearerToken import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import com.tencent.bkrepo.replication.replica.base.impl.remote.base.AuthorizationService @@ -106,10 +103,14 @@ class OciAuthorizationService : AuthorizationService { val httpRequest = HttpUtils.wrapperRequest(property) httpClient.newCall(httpRequest).execute().use { if (!it.isSuccessful) { - val error = JsonUtils.objectMapper.readValue(it.body!!.byteStream(), OciResponse::class.java) + val error = try { + JsonUtils.objectMapper.readValue(it.body!!.byteStream(), OciResponse::class.java).toJsonString() + } catch (ignore: Exception) { + StringPool.EMPTY + } throw ArtifactPushException( "Could not get token from auth service," + - " code is ${it.code} and response is ${error.toJsonString()}" + " code is ${it.code} and response is $error" ) } try { @@ -123,44 +124,6 @@ class OciAuthorizationService : AuthorizationService { } } - /** - * 解析返回头中的WWW_AUTHENTICATE字段, 只针对为Bearer realm - */ - private fun parseWWWAuthenticateHeader(wwwAuthenticate: String, scope: String?): AuthenticationProperty? { - val map: MutableMap = mutableMapOf() - return try { - val params = wwwAuthenticate.split("\",") - params.forEach { - val param = it.split(Regex("="),2) - val name = param.first() - val value = param.last().replace("\"", "") - map[name] = value - } - AuthenticationProperty( - authUrl = map[BEARER_REALM]!!, - service = map[SERVICE]!!, - scope = scope - ) - } catch (e: Exception) { - logger.warn("Parsing wwwAuthenticate header error: ${e.message}") - null - } - } - - private fun buildAuthenticationUrl(property: AuthenticationProperty, userName: String?): String? { - if (property.authUrl.isBlank()) return null - var result = if (property.authUrl.contains(QUESTION)) { - "${property.authUrl}&$SERVICE=${property.service}" - } else { - "${property.authUrl}?$SERVICE=${property.service}" - } - property.scope?.let { - result += "&$SCOPE=${property.scope}" - } - userName?.let { result += "&account=$userName" } - return result - } - companion object { private val logger = LoggerFactory.getLogger(OciAuthorizationService::class.java) } diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt index 082ad9fdbb..4cb6ffba58 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt @@ -33,6 +33,7 @@ import com.tencent.bkrepo.common.api.pojo.ClusterNodeType import com.tencent.bkrepo.common.artifact.event.packages.VersionCreatedEvent import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter.addProtocol import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace import com.tencent.bkrepo.replication.api.ReplicaTaskOperationClient @@ -69,7 +70,6 @@ import com.tencent.bkrepo.replication.service.RemoteNodeService import com.tencent.bkrepo.replication.service.ReplicaNodeDispatchService import com.tencent.bkrepo.replication.service.ReplicaRecordService import com.tencent.bkrepo.replication.service.ReplicaTaskService -import com.tencent.bkrepo.replication.util.HttpUtils.addProtocol import com.tencent.bkrepo.replication.util.ReplicationMetricsRecordUtil.convertToReplicationTaskMetricsRecord import com.tencent.bkrepo.replication.util.ReplicationMetricsRecordUtil.toJson import org.slf4j.LoggerFactory diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt index 914f02e927..1010c53c09 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt @@ -27,19 +27,14 @@ package com.tencent.bkrepo.replication.util -import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.api.constant.HttpHeaders -import com.tencent.bkrepo.common.api.constant.StringPool -import com.tencent.bkrepo.common.storage.innercos.retry -import com.tencent.bkrepo.replication.constant.DELAY_IN_SECONDS -import com.tencent.bkrepo.replication.constant.RETRY_COUNT +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter.addParams import com.tencent.bkrepo.replication.pojo.blob.RequestTag import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import okhttp3.Request import org.springframework.web.bind.annotation.RequestMethod import java.io.IOException import java.net.HttpURLConnection -import java.net.MalformedURLException import java.net.URL object HttpUtils { @@ -51,7 +46,7 @@ object HttpUtils { ): Request { with(requestProperty) { val url = params?.let { - buildUrl(url = requestUrl!!, params = params!!) + addParams(url = requestUrl!!, params = params!!) } ?: requestUrl!! var builder = Request.Builder() .url(url) @@ -73,46 +68,6 @@ object HttpUtils { } } - /** - * 拼接url - */ - fun buildUrl( - url: String, - path: String = StringPool.EMPTY, - params: String = StringPool.EMPTY, - ): String { - val builder = StringBuilder(url) - if (path.isNotBlank()) { - builder.append(CharPool.SLASH).append(path.trimStart(CharPool.SLASH)) - } - if (params.isNotBlank()) { - if (builder.contains(CharPool.QUESTION)) { - builder.append(CharPool.AND).append(params) - } else { - builder.append(CharPool.QUESTION).append(params) - } - } - return builder.toString() - } - - /** - * 针对url如果没传protocol, 则默认以https请求发送 - */ - fun addProtocol(registry: String): URL { - try { - return URL(registry) - } catch (ignore: MalformedURLException) { - } - val url = URL("${StringPool.HTTPS}$registry") - return try { - retry(times = RETRY_COUNT, delayInSeconds = DELAY_IN_SECONDS) { - validateHttpsProtocol(url) - url - } - } catch (ignore: Exception) { - URL(url.toString().replaceFirst("^https".toRegex(), "http")) - } - } /** * Pings a HTTP URL. This effectively sends a HEAD request and returns `true` if the response code is in @@ -139,21 +94,6 @@ object HttpUtils { } } - /** - * 验证registry是否支持https - */ - private fun validateHttpsProtocol(url: URL): Boolean { - return try { - val http: HttpURLConnection = url.openConnection() as HttpURLConnection - http.instanceFollowRedirects = false - http.responseCode - http.disconnect() - true - } catch (e: Exception) { - throw e - } - } - /** * 从Content-Range头中解析出起始位置 */ diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt index d8fabdba8d..45afee4fcc 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt @@ -133,7 +133,7 @@ interface NodeClient { @ApiOperation("删除节点") @DeleteMapping("/batch/delete") - fun deleteNodes(nodesDeleteRequest: NodesDeleteRequest): Response + fun deleteNodes(@RequestBody nodesDeleteRequest: NodesDeleteRequest): Response @ApiOperation("恢复节点") @PostMapping("/restore") @@ -194,4 +194,14 @@ interface NodeClient { @PathVariable repoName: String, @RequestParam fullPath: String ): Response> + + @ApiOperation("通过sha256查询已删除节点") + @GetMapping("/deletedBySha256/detail/{projectId}/{repoName}") + fun getDeletedNodeDetailBySha256( + @PathVariable projectId: String, + @PathVariable repoName: String, + @RequestParam sha256: String + ): Response + + } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt index f6c3c79e34..c954cf223c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt @@ -178,4 +178,10 @@ class NodeController( val artifactInfo = DefaultArtifactInfo(projectId, repoName, fullPath) return ResponseBuilder.success(nodeService.getDeletedNodeDetail(artifactInfo)) } + + override fun getDeletedNodeDetailBySha256( + projectId: String, repoName: String, sha256: String + ): Response { + return ResponseBuilder.success(nodeService.getDeletedNodeDetailBySha256(projectId, repoName, sha256)) + } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt index 2fa41c70bb..617603e44e 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt @@ -48,6 +48,10 @@ interface NodeRestoreOperation { */ fun getDeletedNodeDetail(artifact: ArtifactInfo): List + /** + * 根据 sha256 查询被删除的节点详情 + */ + fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? /** * 恢复被删除的节点 * @return 恢复节点数量 diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt index 4cfe17bc50..b3eb0d925c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt @@ -50,6 +50,7 @@ import com.tencent.bkrepo.repository.util.MetadataUtils import com.tencent.bkrepo.repository.util.NodeQueryHelper import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedFolderQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedPointListQuery +import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedPointListQueryBySha256 import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedPointQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeListQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeQuery @@ -76,6 +77,12 @@ open class NodeRestoreSupport( } } + override fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? { + val query = nodeDeletedPointListQueryBySha256(projectId, repoName, sha256) + val node = nodeDao.findOne(query) + return convertToDetail(node) + } + override fun restoreNode(artifact: ArtifactInfo, nodeRestoreOption: NodeRestoreOption): NodeRestoreResult { with(resolveContext(artifact, nodeRestoreOption)) { return restoreNode(this) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt index 268d705553..1f3fb9ac84 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt @@ -149,6 +149,12 @@ class NodeServiceImpl( return NodeRestoreSupport(this).getDeletedNodeDetail(artifact) } + override fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? { + return NodeRestoreSupport(this).getDeletedNodeDetailBySha256( + projectId, repoName, sha256 + ) + } + @Transactional(rollbackFor = [Throwable::class]) override fun restoreNode(artifact: ArtifactInfo, nodeRestoreOption: NodeRestoreOption): NodeRestoreResult { return NodeRestoreSupport(this).restoreNode(artifact, nodeRestoreOption) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt index 614a9ea66d..34479b7496 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -162,6 +162,10 @@ class EdgeNodeServiceImpl( return NodeRestoreSupport(this).getDeletedNodeDetail(artifact) } + override fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? { + return NodeRestoreSupport(this).getDeletedNodeDetailBySha256(projectId, repoName, sha256) + } + @Transactional(rollbackFor = [Throwable::class]) override fun restoreNode(artifact: ArtifactInfo, nodeRestoreOption: NodeRestoreOption): NodeRestoreResult { centerNodeClient.restoreNode(NodeRestoreRequest(artifact, nodeRestoreOption)) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt index a489df1bb6..e16a1ca508 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt @@ -128,6 +128,17 @@ object NodeQueryHelper { return Query(criteria).with(Sort.by(Sort.Direction.DESC, TNode::deleted.name)) } + /** + * 通过sha256查询被删除节点详情 + */ + fun nodeDeletedPointListQueryBySha256(projectId: String, repoName: String, sha256: String): Query { + val criteria = where(TNode::projectId).isEqualTo(projectId) + .and(TNode::repoName).isEqualTo(repoName) + .and(TNode::sha256).isEqualTo(sha256) + .and(TNode::deleted).ne(null) + return Query(criteria).with(Sort.by(Sort.Direction.DESC, TNode::deleted.name)) + } + /** * 查询单个被删除节点 */ From ce542062abb716c3e861b4c60503d5f800337f96 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Mon, 4 Sep 2023 19:38:17 +0800 Subject: [PATCH 11/11] =?UTF-8?q?feat:=20=E4=B8=80=E6=AC=A1=E6=80=A7?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=94=AF=E6=8C=81=E5=88=86=E5=8F=91=E4=BB=93?= =?UTF-8?q?=E5=BA=93=EF=BC=9B=E6=94=AF=E6=8C=81=E8=87=AA=E5=8A=A8=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E8=BF=9C=E7=A8=8B=E9=A1=B9=E7=9B=AE=E4=BB=93=E5=BA=93?= =?UTF-8?q?=20#1105=20(#1107)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 代码结构调整 #1105 * feat: 增加配置,当远程项目和仓库不存在时是否自动创建;一次性任务支持分发仓库 #1105 * feat: 解决代码冲突#1105 * feat: 代码优化#1105 * feat: 代码优化#1105 * feat: 代码优化#1105 * feat: 去掉无效参数#1105 --- .../request/RemoteConfigCreateRequest.kt | 3 + .../request/RemoteConfigUpdateRequest.kt | 3 + .../request/RemoteRunOnceTaskCreateRequest.kt | 3 + .../pojo/task/setting/ReplicaSetting.kt | 6 +- .../cluster/ClusterReplicaTaskController.kt | 2 +- .../service/ArtifactPushController.kt | 2 +- .../exception/ArtifactContentReadException.kt | 2 +- .../exception/ArtifactPushException.kt | 2 +- .../exception/ArtifactSourceCheckException.kt | 2 +- .../exception/RegexCheckException.kt | 2 +- .../replication/metrics/ReplicationMetrics.kt | 12 +- .../progress/ProgressInterceptor.kt | 2 +- .../progress/ProgressRequestBody.kt | 2 +- .../{base => }/context/FilePushContext.kt | 2 +- .../{base => }/context/ReplicaContext.kt | 18 +- .../context/ReplicaExecutionContext.kt | 2 +- .../executor/AbstractReplicaJobExecutor.kt | 8 +- .../executor/EdgePullThreadPoolExecutor.kt | 2 +- .../EventConsumerThreadPoolExecutor.kt | 2 +- .../executor/ManualThreadPoolExecutor.kt | 2 +- .../executor/OciThreadPoolExecutor.kt | 2 +- .../executor/ReplicaThreadPoolExecutor.kt | 4 +- .../executor/RunOnceThreadPoolExecutor.kt | 2 +- .../{base => }/replicator/Replicator.kt | 6 +- .../base}/ArtifactReplicationHandler.kt | 7 +- .../ClusterArtifactReplicationHandler.kt | 7 +- ...RemoteClusterArtifactReplicationHandler.kt | 7 +- .../commitedge/CenterClusterReplicator.kt | 10 +- .../commitedge/CenterRemoteReplicator.kt | 8 +- .../commitedge/EdgeReplicaContextHolder.kt | 4 +- .../standalone}/ClusterReplicator.kt | 13 +- .../standalone}/EdgeNodeReplicator.kt | 13 +- .../standalone}/RemoteReplicator.kt | 11 +- .../internal/PackageNodeMappings.kt | 12 +- .../internal/type/DockerPackageNodeMapper.kt | 29 +- .../internal/type/MavenPackageNodeMapper.kt | 4 +- .../internal/type/NpmPackageNodeMapper.kt | 4 +- .../internal/type/PackageNodeMapper.kt | 4 +- .../remote/ArtifactPushMappings.kt | 10 +- .../remote/base/AuthorizationService.kt | 2 +- .../remote/base/PushClient.kt | 4 +- .../type/helm/HelmArtifactPushClient.kt | 8 +- .../type/helm/HelmAuthorizationService.kt | 4 +- .../remote/type/oci/OciArtifactPushClient.kt | 16 +- .../type/oci/OciAuthorizationService.kt | 6 +- .../{base => type}/AbstractReplicaService.kt | 55 ++- .../replica/{base => type}/ReplicaService.kt | 6 +- .../{ => type}/edge/EdgeNodeReportJob.kt | 4 +- .../edge/EdgePullReplicaExecutor.kt | 6 +- .../edge/EdgePullReplicaTaskEvent.kt | 2 +- .../{ => type}/edge/EdgePullReplicaTaskJob.kt | 6 +- .../{ => type}/edge/EdgeReplicaTaskJob.kt | 10 +- .../{ => type}/event/ArtifactEventConsumer.kt | 6 +- .../event/BootAssemblyEventListener.kt | 4 +- .../event/EventBasedReplicaJobExecutor.kt | 6 +- .../event/EventBasedReplicaService.kt | 12 +- .../replica/{ => type}/event/EventConsumer.kt | 2 +- .../manual/ManualBasedReplicaService.kt | 101 ++++-- .../manual/ManualReplicaJobExecutor.kt | 4 +- .../schedule/ReplicaTaskScheduler.kt | 4 +- .../schedule/ScheduledReplicaJob.kt | 4 +- .../schedule/ScheduledReplicaJobExecutor.kt | 6 +- .../schedule/ScheduledReplicaService.kt | 30 +- .../{ => type}/schedule/TaskReloadManager.kt | 8 +- .../service/EdgeReplicaTaskRecordService.kt | 2 +- .../impl/EdgeReplicaTaskRecordServiceImpl.kt | 2 +- .../service/impl/RemoteNodeServiceImpl.kt | 327 +++++++++++------- .../service/impl/ReplicaExtServiceImpl.kt | 6 +- .../service/impl/ReplicaTaskServiceImpl.kt | 10 +- .../base/handler => util}/DefaultHandler.kt | 5 +- .../base => util}/OkHttpClientPool.kt | 29 +- .../bkrepo/replication/QuartzJobReloadTest.kt | 4 +- .../service/TaskReloadManagerTest.kt | 6 +- 73 files changed, 575 insertions(+), 368 deletions(-) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/{replica/base/impl/remote => }/exception/ArtifactContentReadException.kt (95%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/{replica/base/impl/remote => }/exception/ArtifactPushException.kt (95%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/{replica/base/impl/remote => }/exception/ArtifactSourceCheckException.kt (95%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/{replica/base/impl/remote => }/exception/RegexCheckException.kt (95%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/context/FilePushContext.kt (96%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/context/ReplicaContext.kt (91%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/context/ReplicaExecutionContext.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/executor/AbstractReplicaJobExecutor.kt (94%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/executor/EdgePullThreadPoolExecutor.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/executor/EventConsumerThreadPoolExecutor.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/executor/ManualThreadPoolExecutor.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/executor/OciThreadPoolExecutor.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/executor/ReplicaThreadPoolExecutor.kt (94%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/executor/RunOnceThreadPoolExecutor.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/replicator/Replicator.kt (93%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/handler => replicator/base}/ArtifactReplicationHandler.kt (98%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/handler => replicator/base/internal}/ClusterArtifactReplicationHandler.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/handler => replicator/base/remote}/RemoteClusterArtifactReplicationHandler.kt (93%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/replicator/commitedge/CenterClusterReplicator.kt (91%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/replicator/commitedge/CenterRemoteReplicator.kt (91%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => }/replicator/commitedge/EdgeReplicaContextHolder.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/replicator => replicator/standalone}/ClusterReplicator.kt (96%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/replicator => replicator/standalone}/EdgeNodeReplicator.kt (91%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/replicator => replicator/standalone}/RemoteReplicator.kt (93%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/internal/PackageNodeMappings.kt (84%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/internal/type/DockerPackageNodeMapper.kt (74%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/internal/type/MavenPackageNodeMapper.kt (95%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/internal/type/NpmPackageNodeMapper.kt (94%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/internal/type/PackageNodeMapper.kt (93%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/remote/ArtifactPushMappings.kt (87%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/remote/base/AuthorizationService.kt (96%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/remote/base/PushClient.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/remote/type/helm/HelmArtifactPushClient.kt (96%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/remote/type/helm/HelmAuthorizationService.kt (92%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/remote/type/oci/OciArtifactPushClient.kt (95%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base/impl => repository}/remote/type/oci/OciAuthorizationService.kt (95%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => type}/AbstractReplicaService.kt (90%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{base => type}/ReplicaService.kt (89%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/edge/EdgeNodeReportJob.kt (96%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/edge/EdgePullReplicaExecutor.kt (98%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/edge/EdgePullReplicaTaskEvent.kt (96%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/edge/EdgePullReplicaTaskJob.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/edge/EdgeReplicaTaskJob.kt (96%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/event/ArtifactEventConsumer.kt (92%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/event/BootAssemblyEventListener.kt (95%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/event/EventBasedReplicaJobExecutor.kt (93%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/event/EventBasedReplicaService.kt (90%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/event/EventConsumer.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/manual/ManualBasedReplicaService.kt (51%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/manual/ManualReplicaJobExecutor.kt (97%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/schedule/ReplicaTaskScheduler.kt (96%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/schedule/ScheduledReplicaJob.kt (95%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/schedule/ScheduledReplicaJobExecutor.kt (96%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/schedule/ScheduledReplicaService.kt (70%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/{ => type}/schedule/TaskReloadManager.kt (94%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/{replica/base/handler => util}/DefaultHandler.kt (95%) rename src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/{replica/base => util}/OkHttpClientPool.kt (50%) diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteConfigCreateRequest.kt b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteConfigCreateRequest.kt index fb247fa33e..cfe6a9eeac 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteConfigCreateRequest.kt +++ b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteConfigCreateRequest.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.replication.pojo.remote.request +import com.tencent.bkrepo.replication.pojo.request.ReplicaObjectType import com.tencent.bkrepo.replication.pojo.request.ReplicaType import com.tencent.bkrepo.replication.pojo.task.objects.PackageConstraint import com.tencent.bkrepo.replication.pojo.task.objects.PathConstraint @@ -61,6 +62,8 @@ data class RemoteConfigCreateRequest( val remoteProjectId: String? = null, @ApiModelProperty("远程仓库") val remoteRepoName: String? = null, + @ApiModelProperty("同步对象类型, 只支持clusterId存在的情况", required = true) + val replicaObjectType: ReplicaObjectType? = null, @ApiModelProperty("包限制条件", required = false) val packageConstraints: List? = null, @ApiModelProperty("路径限制条件", required = false) diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteConfigUpdateRequest.kt b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteConfigUpdateRequest.kt index b8c3aba555..369bd34408 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteConfigUpdateRequest.kt +++ b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteConfigUpdateRequest.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.replication.pojo.remote.request +import com.tencent.bkrepo.replication.pojo.request.ReplicaObjectType import com.tencent.bkrepo.replication.pojo.request.ReplicaType import com.tencent.bkrepo.replication.pojo.task.objects.PackageConstraint import com.tencent.bkrepo.replication.pojo.task.objects.PathConstraint @@ -59,6 +60,8 @@ data class RemoteConfigUpdateRequest( val remoteProjectId: String? = null, @ApiModelProperty("远程仓库") val remoteRepoName: String? = null, + @ApiModelProperty("同步对象类型, 只支持clusterId存在的情况", required = true) + val replicaObjectType: ReplicaObjectType? = null, @ApiModelProperty("包限制条件", required = false) val packageConstraints: List? = null, @ApiModelProperty("路径限制条件", required = false) diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteRunOnceTaskCreateRequest.kt b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteRunOnceTaskCreateRequest.kt index 031c81dd9e..0253e8a1aa 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteRunOnceTaskCreateRequest.kt +++ b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/request/RemoteRunOnceTaskCreateRequest.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.replication.pojo.remote.request +import com.tencent.bkrepo.replication.pojo.request.ReplicaObjectType import com.tencent.bkrepo.replication.pojo.request.ReplicaType import com.tencent.bkrepo.replication.pojo.task.setting.ReplicaSetting import io.swagger.annotations.ApiModel @@ -57,6 +58,8 @@ data class RemoteRunOnceTaskCreateRequest( val remoteProjectId: String? = null, @ApiModelProperty("远程仓库") val remoteRepoName: String? = null, + @ApiModelProperty("同步对象类型, 只支持clusterId存在的情况", required = true) + val replicaObjectType: ReplicaObjectType? = null, @ApiModelProperty("包名", required = false) val packageName: String? = null, @ApiModelProperty("包版本", required = false) diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/task/setting/ReplicaSetting.kt b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/task/setting/ReplicaSetting.kt index 7894fdb293..0861922d9d 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/task/setting/ReplicaSetting.kt +++ b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/task/setting/ReplicaSetting.kt @@ -58,5 +58,9 @@ data class ReplicaSetting( /** * 是否校验文件存储一致性 */ - val storageConsistencyCheck: Boolean = false + val storageConsistencyCheck: Boolean = false, + /** + * 当远程项目或者仓库不存在时是否自动创建 + */ + val automaticCreateRemoteRepo: Boolean = true ) diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/cluster/ClusterReplicaTaskController.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/cluster/ClusterReplicaTaskController.kt index e186796cc5..42efbad60e 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/cluster/ClusterReplicaTaskController.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/cluster/ClusterReplicaTaskController.kt @@ -36,7 +36,7 @@ import com.tencent.bkrepo.replication.pojo.request.ReplicaType import com.tencent.bkrepo.replication.pojo.task.EdgeReplicaTaskRecord import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskInfo import com.tencent.bkrepo.replication.pojo.task.objects.ReplicaObjectInfo -import com.tencent.bkrepo.replication.replica.base.replicator.commitedge.EdgeReplicaContextHolder +import com.tencent.bkrepo.replication.replica.replicator.commitedge.EdgeReplicaContextHolder import com.tencent.bkrepo.replication.service.EdgeReplicaTaskRecordService import com.tencent.bkrepo.replication.service.ReplicaTaskService import org.springframework.web.bind.annotation.RestController diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactPushController.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactPushController.kt index 2d92d999eb..8cb7b0145c 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactPushController.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactPushController.kt @@ -37,7 +37,7 @@ import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.replication.api.ArtifactPushClient import com.tencent.bkrepo.replication.pojo.remote.request.ArtifactPushRequest -import com.tencent.bkrepo.replication.replica.event.EventBasedReplicaJobExecutor +import com.tencent.bkrepo.replication.replica.type.event.EventBasedReplicaJobExecutor import com.tencent.bkrepo.replication.service.ReplicaTaskService import org.slf4j.LoggerFactory import org.springframework.web.bind.annotation.RestController diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/ArtifactContentReadException.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/ArtifactContentReadException.kt similarity index 95% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/ArtifactContentReadException.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/ArtifactContentReadException.kt index d8bb2629d8..eb60898cd9 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/ArtifactContentReadException.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/ArtifactContentReadException.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.remote.exception +package com.tencent.bkrepo.replication.exception /** * 文件内容读取报错 diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/ArtifactPushException.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/ArtifactPushException.kt similarity index 95% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/ArtifactPushException.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/ArtifactPushException.kt index 90c8820a20..9a1125cd5d 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/ArtifactPushException.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/ArtifactPushException.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.remote.exception +package com.tencent.bkrepo.replication.exception /** * 推送到远端集群异常 */ diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/ArtifactSourceCheckException.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/ArtifactSourceCheckException.kt similarity index 95% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/ArtifactSourceCheckException.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/ArtifactSourceCheckException.kt index e019b957ab..72912e87f0 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/ArtifactSourceCheckException.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/ArtifactSourceCheckException.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.remote.exception +package com.tencent.bkrepo.replication.exception /** * Artifact来源异常 diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/RegexCheckException.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/RegexCheckException.kt similarity index 95% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/RegexCheckException.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/RegexCheckException.kt index 7b11d95aa4..31d44a4052 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/exception/RegexCheckException.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/exception/RegexCheckException.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.remote.exception +package com.tencent.bkrepo.replication.exception /** * 规则校验异常 diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/metrics/ReplicationMetrics.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/metrics/ReplicationMetrics.kt index 3e85db7e59..535ebda315 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/metrics/ReplicationMetrics.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/metrics/ReplicationMetrics.kt @@ -51,12 +51,12 @@ import com.tencent.bkrepo.replication.constant.RUN_ONCE_EXECUTOR_ACTIVE_COUNT import com.tencent.bkrepo.replication.constant.RUN_ONCE_EXECUTOR_ACTIVE_COUNT_DESC import com.tencent.bkrepo.replication.constant.RUN_ONCE_EXECUTOR_QUEUE_SIZE import com.tencent.bkrepo.replication.constant.RUN_ONCE_EXECUTOR_QUEUE_SIZE_DESC -import com.tencent.bkrepo.replication.replica.base.executor.EdgePullThreadPoolExecutor -import com.tencent.bkrepo.replication.replica.base.executor.EventConsumerThreadPoolExecutor -import com.tencent.bkrepo.replication.replica.base.executor.ManualThreadPoolExecutor -import com.tencent.bkrepo.replication.replica.base.executor.OciThreadPoolExecutor -import com.tencent.bkrepo.replication.replica.base.executor.ReplicaThreadPoolExecutor -import com.tencent.bkrepo.replication.replica.base.executor.RunOnceThreadPoolExecutor +import com.tencent.bkrepo.replication.replica.executor.EdgePullThreadPoolExecutor +import com.tencent.bkrepo.replication.replica.executor.EventConsumerThreadPoolExecutor +import com.tencent.bkrepo.replication.replica.executor.ManualThreadPoolExecutor +import com.tencent.bkrepo.replication.replica.executor.OciThreadPoolExecutor +import com.tencent.bkrepo.replication.replica.executor.ReplicaThreadPoolExecutor +import com.tencent.bkrepo.replication.replica.executor.RunOnceThreadPoolExecutor import io.micrometer.core.instrument.Gauge import io.micrometer.core.instrument.MeterRegistry import io.micrometer.core.instrument.binder.MeterBinder diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/interceptor/progress/ProgressInterceptor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/interceptor/progress/ProgressInterceptor.kt index 42f17317be..d2bd39c4a6 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/interceptor/progress/ProgressInterceptor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/interceptor/progress/ProgressInterceptor.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/interceptor/progress/ProgressRequestBody.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/interceptor/progress/ProgressRequestBody.kt index f5b1b97746..d6956d9bd3 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/interceptor/progress/ProgressRequestBody.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/interceptor/progress/ProgressRequestBody.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/context/FilePushContext.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/context/FilePushContext.kt similarity index 96% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/context/FilePushContext.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/context/FilePushContext.kt index 3020cc4206..c7ed69af26 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/context/FilePushContext.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/context/FilePushContext.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.context +package com.tencent.bkrepo.replication.replica.context import com.tencent.bkrepo.common.api.pojo.Response import okhttp3.OkHttpClient diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/context/ReplicaContext.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/context/ReplicaContext.kt similarity index 91% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/context/ReplicaContext.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/context/ReplicaContext.kt index 5d9836b9f4..c0153585ad 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/context/ReplicaContext.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/context/ReplicaContext.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.context +package com.tencent.bkrepo.replication.replica.context import com.tencent.bkrepo.common.api.pojo.ClusterArchitecture import com.tencent.bkrepo.common.api.pojo.ClusterNodeType @@ -44,15 +44,15 @@ import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus import com.tencent.bkrepo.replication.pojo.record.ReplicaRecordInfo import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskDetail import com.tencent.bkrepo.replication.pojo.task.objects.ReplicaObjectInfo -import com.tencent.bkrepo.replication.replica.base.OkHttpClientPool import com.tencent.bkrepo.replication.replica.base.interceptor.RetryInterceptor import com.tencent.bkrepo.replication.replica.base.interceptor.SignInterceptor -import com.tencent.bkrepo.replication.replica.base.replicator.ClusterReplicator -import com.tencent.bkrepo.replication.replica.base.replicator.EdgeNodeReplicator -import com.tencent.bkrepo.replication.replica.base.replicator.RemoteReplicator -import com.tencent.bkrepo.replication.replica.base.replicator.Replicator -import com.tencent.bkrepo.replication.replica.base.replicator.commitedge.CenterClusterReplicator -import com.tencent.bkrepo.replication.replica.base.replicator.commitedge.CenterRemoteReplicator +import com.tencent.bkrepo.replication.replica.replicator.Replicator +import com.tencent.bkrepo.replication.replica.replicator.commitedge.CenterClusterReplicator +import com.tencent.bkrepo.replication.replica.replicator.commitedge.CenterRemoteReplicator +import com.tencent.bkrepo.replication.replica.replicator.standalone.ClusterReplicator +import com.tencent.bkrepo.replication.replica.replicator.standalone.EdgeNodeReplicator +import com.tencent.bkrepo.replication.replica.replicator.standalone.RemoteReplicator +import com.tencent.bkrepo.replication.util.OkHttpClientPool import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import okhttp3.OkHttpClient import org.slf4j.LoggerFactory diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/context/ReplicaExecutionContext.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/context/ReplicaExecutionContext.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/context/ReplicaExecutionContext.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/context/ReplicaExecutionContext.kt index 3f0f5c8ed9..ceafe53d81 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/context/ReplicaExecutionContext.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/context/ReplicaExecutionContext.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.context +package com.tencent.bkrepo.replication.replica.context import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus import com.tencent.bkrepo.replication.pojo.record.ReplicaProgress diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/AbstractReplicaJobExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/AbstractReplicaJobExecutor.kt similarity index 94% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/AbstractReplicaJobExecutor.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/AbstractReplicaJobExecutor.kt index 1f2ce54e34..3a27b84c6a 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/AbstractReplicaJobExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/AbstractReplicaJobExecutor.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.executor +package com.tencent.bkrepo.replication.replica.executor import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace @@ -36,8 +36,8 @@ import com.tencent.bkrepo.replication.pojo.record.ExecutionResult import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus import com.tencent.bkrepo.replication.pojo.record.ReplicaRecordInfo import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskDetail -import com.tencent.bkrepo.replication.replica.base.ReplicaService -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.type.ReplicaService +import com.tencent.bkrepo.replication.replica.context.ReplicaContext import com.tencent.bkrepo.replication.service.ClusterNodeService import org.slf4j.LoggerFactory import java.util.concurrent.Callable diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/EdgePullThreadPoolExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/EdgePullThreadPoolExecutor.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/EdgePullThreadPoolExecutor.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/EdgePullThreadPoolExecutor.kt index 62b95266a6..1166532036 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/EdgePullThreadPoolExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/EdgePullThreadPoolExecutor.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.executor +package com.tencent.bkrepo.replication.replica.executor import com.google.common.util.concurrent.ThreadFactoryBuilder import java.util.concurrent.SynchronousQueue diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/EventConsumerThreadPoolExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/EventConsumerThreadPoolExecutor.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/EventConsumerThreadPoolExecutor.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/EventConsumerThreadPoolExecutor.kt index 187db3993c..9491e52db6 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/EventConsumerThreadPoolExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/EventConsumerThreadPoolExecutor.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.executor +package com.tencent.bkrepo.replication.replica.executor import com.google.common.util.concurrent.ThreadFactoryBuilder import java.util.concurrent.ArrayBlockingQueue diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/ManualThreadPoolExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/ManualThreadPoolExecutor.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/ManualThreadPoolExecutor.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/ManualThreadPoolExecutor.kt index 547fadb77f..97cd568e73 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/ManualThreadPoolExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/ManualThreadPoolExecutor.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.executor +package com.tencent.bkrepo.replication.replica.executor import com.google.common.util.concurrent.ThreadFactoryBuilder import java.util.concurrent.ArrayBlockingQueue diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/OciThreadPoolExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/OciThreadPoolExecutor.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/OciThreadPoolExecutor.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/OciThreadPoolExecutor.kt index 942688b166..02994d223e 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/OciThreadPoolExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/OciThreadPoolExecutor.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.executor +package com.tencent.bkrepo.replication.replica.executor import com.google.common.util.concurrent.ThreadFactoryBuilder import java.util.concurrent.ArrayBlockingQueue diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/ReplicaThreadPoolExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/ReplicaThreadPoolExecutor.kt similarity index 94% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/ReplicaThreadPoolExecutor.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/ReplicaThreadPoolExecutor.kt index 2890476375..e973c4367b 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/ReplicaThreadPoolExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/ReplicaThreadPoolExecutor.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.executor +package com.tencent.bkrepo.replication.replica.executor import com.google.common.util.concurrent.ThreadFactoryBuilder import java.util.concurrent.ArrayBlockingQueue diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/RunOnceThreadPoolExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/RunOnceThreadPoolExecutor.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/RunOnceThreadPoolExecutor.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/RunOnceThreadPoolExecutor.kt index 9e021eb558..fe95935b83 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/executor/RunOnceThreadPoolExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/executor/RunOnceThreadPoolExecutor.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.executor +package com.tencent.bkrepo.replication.replica.executor import com.google.common.util.concurrent.ThreadFactoryBuilder import java.util.concurrent.ArrayBlockingQueue diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/Replicator.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/Replicator.kt similarity index 93% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/Replicator.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/Replicator.kt index 280b64aaed..499ba86739 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/Replicator.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/Replicator.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,9 +25,9 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.replicator +package com.tencent.bkrepo.replication.replica.replicator -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.context.ReplicaContext import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/base/ArtifactReplicationHandler.kt similarity index 98% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/base/ArtifactReplicationHandler.kt index ee395dce11..5ef8ffeecd 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/base/ArtifactReplicationHandler.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.handler +package com.tencent.bkrepo.replication.replica.replicator.base import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.MediaTypes @@ -43,8 +43,9 @@ import com.tencent.bkrepo.replication.pojo.blob.RequestTag import com.tencent.bkrepo.replication.pojo.remote.DefaultHandlerResult import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import com.tencent.bkrepo.replication.pojo.request.ReplicaType -import com.tencent.bkrepo.replication.replica.base.context.FilePushContext -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.context.FilePushContext +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.util.DefaultHandler import com.tencent.bkrepo.replication.util.StreamRequestBody import okhttp3.Headers import okhttp3.MediaType.Companion.toMediaTypeOrNull diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ClusterArtifactReplicationHandler.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/base/internal/ClusterArtifactReplicationHandler.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ClusterArtifactReplicationHandler.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/base/internal/ClusterArtifactReplicationHandler.kt index 278bf09dc9..0c731facad 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ClusterArtifactReplicationHandler.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/base/internal/ClusterArtifactReplicationHandler.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.handler +package com.tencent.bkrepo.replication.replica.replicator.base.internal import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.StringPool @@ -40,12 +40,13 @@ import com.tencent.bkrepo.replication.constant.SHA256 import com.tencent.bkrepo.replication.constant.SIZE import com.tencent.bkrepo.replication.constant.STORAGE_KEY import com.tencent.bkrepo.replication.enums.WayOfPushArtifact +import com.tencent.bkrepo.replication.exception.ArtifactPushException import com.tencent.bkrepo.replication.fdtp.FdtpAFTClientFactory import com.tencent.bkrepo.replication.fdtp.FdtpServerProperties import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.blob.RequestTag -import com.tencent.bkrepo.replication.replica.base.context.FilePushContext -import com.tencent.bkrepo.replication.replica.base.impl.remote.exception.ArtifactPushException +import com.tencent.bkrepo.replication.replica.context.FilePushContext +import com.tencent.bkrepo.replication.replica.replicator.base.ArtifactReplicationHandler import com.tencent.bkrepo.replication.replica.base.process.ProgressListener import com.tencent.bkrepo.replication.util.StreamRequestBody import io.netty.channel.ChannelProgressiveFuture diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/base/remote/RemoteClusterArtifactReplicationHandler.kt similarity index 93% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/base/remote/RemoteClusterArtifactReplicationHandler.kt index 8dec006a3b..0a7cd313fb 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/base/remote/RemoteClusterArtifactReplicationHandler.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.handler +package com.tencent.bkrepo.replication.replica.replicator.base.remote import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter @@ -33,8 +33,9 @@ import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.constant.OCI_BLOBS_UPLOAD_FIRST_STEP_URL import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.remote.DefaultHandlerResult -import com.tencent.bkrepo.replication.replica.base.context.FilePushContext -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.context.FilePushContext +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.replicator.base.ArtifactReplicationHandler import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.net.URL diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/commitedge/CenterClusterReplicator.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterClusterReplicator.kt similarity index 91% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/commitedge/CenterClusterReplicator.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterClusterReplicator.kt index f44831e2bf..6c28ab2e6a 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/commitedge/CenterClusterReplicator.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterClusterReplicator.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,15 +25,15 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.replicator.commitedge +package com.tencent.bkrepo.replication.replica.replicator.commitedge import com.tencent.bkrepo.common.service.cluster.ClusterProperties import com.tencent.bkrepo.common.service.cluster.CommitEdgeCenterCondition import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.replica.base.handler.ClusterArtifactReplicationHandler -import com.tencent.bkrepo.replication.replica.base.replicator.ClusterReplicator +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.replicator.base.internal.ClusterArtifactReplicationHandler +import com.tencent.bkrepo.replication.replica.replicator.standalone.ClusterReplicator import com.tencent.bkrepo.replication.service.EdgeReplicaTaskRecordService import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/commitedge/CenterRemoteReplicator.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterRemoteReplicator.kt similarity index 91% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/commitedge/CenterRemoteReplicator.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterRemoteReplicator.kt index 0c88e83fab..58e5ff6be6 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/commitedge/CenterRemoteReplicator.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterRemoteReplicator.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,12 +25,12 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.replicator.commitedge +package com.tencent.bkrepo.replication.replica.replicator.commitedge import com.tencent.bkrepo.common.service.cluster.CommitEdgeCenterCondition import com.tencent.bkrepo.replication.config.ReplicationProperties -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.replica.base.replicator.RemoteReplicator +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.replicator.standalone.RemoteReplicator import com.tencent.bkrepo.replication.service.EdgeReplicaTaskRecordService import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/commitedge/EdgeReplicaContextHolder.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/EdgeReplicaContextHolder.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/commitedge/EdgeReplicaContextHolder.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/EdgeReplicaContextHolder.kt index 3afe6f96f9..025a161444 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/commitedge/EdgeReplicaContextHolder.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/EdgeReplicaContextHolder.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.replicator.commitedge +package com.tencent.bkrepo.replication.replica.replicator.commitedge import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.net.speedtest.Counter diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/ClusterReplicator.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/standalone/ClusterReplicator.kt similarity index 96% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/ClusterReplicator.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/standalone/ClusterReplicator.kt index 903dc4cf1a..47d5e37d12 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/ClusterReplicator.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/standalone/ClusterReplicator.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.replicator +package com.tencent.bkrepo.replication.replica.replicator.standalone import com.google.common.base.Throwables import com.tencent.bkrepo.common.api.constant.HttpStatus @@ -39,14 +39,15 @@ import com.tencent.bkrepo.replication.constant.DEFAULT_VERSION import com.tencent.bkrepo.replication.constant.DELAY_IN_SECONDS import com.tencent.bkrepo.replication.constant.RETRY_COUNT import com.tencent.bkrepo.replication.enums.WayOfPushArtifact +import com.tencent.bkrepo.replication.exception.ArtifactPushException import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.request.PackageVersionExistCheckRequest import com.tencent.bkrepo.replication.pojo.task.setting.ConflictStrategy -import com.tencent.bkrepo.replication.replica.base.context.FilePushContext -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.replica.base.handler.ClusterArtifactReplicationHandler -import com.tencent.bkrepo.replication.replica.base.impl.internal.PackageNodeMappings -import com.tencent.bkrepo.replication.replica.base.impl.remote.exception.ArtifactPushException +import com.tencent.bkrepo.replication.replica.replicator.base.internal.ClusterArtifactReplicationHandler +import com.tencent.bkrepo.replication.replica.repository.internal.PackageNodeMappings +import com.tencent.bkrepo.replication.replica.context.FilePushContext +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.replicator.Replicator import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/EdgeNodeReplicator.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/standalone/EdgeNodeReplicator.kt similarity index 91% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/EdgeNodeReplicator.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/standalone/EdgeNodeReplicator.kt index 3cf3e7c0ea..661e065159 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/EdgeNodeReplicator.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/standalone/EdgeNodeReplicator.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.replicator +package com.tencent.bkrepo.replication.replica.replicator.standalone import com.google.common.base.Throwables import com.tencent.bkrepo.common.api.constant.HttpStatus @@ -34,12 +34,13 @@ import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.constant.DELAY_IN_SECONDS import com.tencent.bkrepo.replication.constant.RETRY_COUNT import com.tencent.bkrepo.replication.enums.WayOfPushArtifact +import com.tencent.bkrepo.replication.exception.ArtifactPushException import com.tencent.bkrepo.replication.manager.LocalDataManager -import com.tencent.bkrepo.replication.replica.base.context.FilePushContext -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.replica.base.handler.ClusterArtifactReplicationHandler -import com.tencent.bkrepo.replication.replica.base.impl.internal.PackageNodeMappings -import com.tencent.bkrepo.replication.replica.base.impl.remote.exception.ArtifactPushException +import com.tencent.bkrepo.replication.replica.replicator.base.internal.ClusterArtifactReplicationHandler +import com.tencent.bkrepo.replication.replica.repository.internal.PackageNodeMappings +import com.tencent.bkrepo.replication.replica.context.FilePushContext +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.replicator.Replicator import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/RemoteReplicator.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/standalone/RemoteReplicator.kt similarity index 93% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/RemoteReplicator.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/standalone/RemoteReplicator.kt index 6f65ec3b76..ff97e31b09 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/replicator/RemoteReplicator.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/standalone/RemoteReplicator.kt @@ -25,14 +25,15 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.replicator +package com.tencent.bkrepo.replication.replica.replicator.standalone import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.replica.base.impl.remote.ArtifactPushMappings -import com.tencent.bkrepo.replication.replica.base.impl.remote.exception.ArtifactSourceCheckException -import com.tencent.bkrepo.replication.replica.base.impl.remote.exception.RegexCheckException +import com.tencent.bkrepo.replication.exception.ArtifactSourceCheckException +import com.tencent.bkrepo.replication.exception.RegexCheckException +import com.tencent.bkrepo.replication.replica.repository.remote.ArtifactPushMappings +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.replicator.Replicator import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/PackageNodeMappings.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/PackageNodeMappings.kt similarity index 84% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/PackageNodeMappings.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/PackageNodeMappings.kt index 152fc3ea89..81e85a72dc 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/PackageNodeMappings.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/PackageNodeMappings.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,14 +25,14 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.internal +package com.tencent.bkrepo.replication.replica.repository.internal import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.service.util.SpringContextUtils -import com.tencent.bkrepo.replication.replica.base.impl.internal.type.DockerPackageNodeMapper -import com.tencent.bkrepo.replication.replica.base.impl.internal.type.MavenPackageNodeMapper -import com.tencent.bkrepo.replication.replica.base.impl.internal.type.NpmPackageNodeMapper -import com.tencent.bkrepo.replication.replica.base.impl.internal.type.PackageNodeMapper +import com.tencent.bkrepo.replication.replica.repository.internal.type.DockerPackageNodeMapper +import com.tencent.bkrepo.replication.replica.repository.internal.type.MavenPackageNodeMapper +import com.tencent.bkrepo.replication.replica.repository.internal.type.NpmPackageNodeMapper +import com.tencent.bkrepo.replication.replica.repository.internal.type.PackageNodeMapper import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/DockerPackageNodeMapper.kt similarity index 74% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/DockerPackageNodeMapper.kt index bfb0450916..a55bbb2ff7 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/DockerPackageNodeMapper.kt @@ -1,4 +1,31 @@ -package com.tencent.bkrepo.replication.replica.base.impl.internal.type +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.replication.replica.repository.internal.type import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException import com.tencent.bkrepo.common.artifact.pojo.RepositoryType diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/MavenPackageNodeMapper.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/MavenPackageNodeMapper.kt similarity index 95% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/MavenPackageNodeMapper.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/MavenPackageNodeMapper.kt index d533a97aa8..1e9195be76 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/MavenPackageNodeMapper.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/MavenPackageNodeMapper.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.internal.type +package com.tencent.bkrepo.replication.replica.repository.internal.type import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.repository.api.NodeClient diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/NpmPackageNodeMapper.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/NpmPackageNodeMapper.kt similarity index 94% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/NpmPackageNodeMapper.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/NpmPackageNodeMapper.kt index b825d0fedf..2dcdbc28ba 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/NpmPackageNodeMapper.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/NpmPackageNodeMapper.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.internal.type +package com.tencent.bkrepo.replication.replica.repository.internal.type import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.util.PackageKeys diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/PackageNodeMapper.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/PackageNodeMapper.kt similarity index 93% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/PackageNodeMapper.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/PackageNodeMapper.kt index ff9b4522b3..3a627fa251 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/PackageNodeMapper.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/internal/type/PackageNodeMapper.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.internal.type +package com.tencent.bkrepo.replication.replica.repository.internal.type import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.repository.pojo.packages.PackageSummary diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/ArtifactPushMappings.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/ArtifactPushMappings.kt similarity index 87% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/ArtifactPushMappings.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/ArtifactPushMappings.kt index 7c52558042..ba42e5bd41 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/ArtifactPushMappings.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/ArtifactPushMappings.kt @@ -25,14 +25,14 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.remote +package com.tencent.bkrepo.replication.replica.repository.remote import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.service.util.SpringContextUtils -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.replica.base.impl.remote.base.PushClient -import com.tencent.bkrepo.replication.replica.base.impl.remote.type.helm.HelmArtifactPushClient -import com.tencent.bkrepo.replication.replica.base.impl.remote.type.oci.OciArtifactPushClient +import com.tencent.bkrepo.replication.replica.repository.remote.base.PushClient +import com.tencent.bkrepo.replication.replica.repository.remote.type.helm.HelmArtifactPushClient +import com.tencent.bkrepo.replication.replica.repository.remote.type.oci.OciArtifactPushClient +import com.tencent.bkrepo.replication.replica.context.ReplicaContext import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/base/AuthorizationService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/base/AuthorizationService.kt similarity index 96% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/base/AuthorizationService.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/base/AuthorizationService.kt index 6579d66fa1..b2abc9a4ae 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/base/AuthorizationService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/base/AuthorizationService.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.remote.base +package com.tencent.bkrepo.replication.replica.repository.remote.base import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import okhttp3.OkHttpClient diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/base/PushClient.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/base/PushClient.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/base/PushClient.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/base/PushClient.kt index dd5c60a7cc..ecc14e9aee 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/base/PushClient.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/base/PushClient.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.remote.base +package com.tencent.bkrepo.replication.replica.repository.remote.base import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.service.cluster.ClusterInfo @@ -34,9 +34,9 @@ import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.blob.RequestTag import com.tencent.bkrepo.replication.pojo.request.ReplicaType -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext import com.tencent.bkrepo.replication.replica.base.interceptor.RetryInterceptor import com.tencent.bkrepo.replication.replica.base.interceptor.progress.ProgressInterceptor +import com.tencent.bkrepo.replication.replica.context.ReplicaContext import com.tencent.bkrepo.repository.pojo.node.NodeDetail import okhttp3.OkHttpClient import okhttp3.Protocol diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/helm/HelmArtifactPushClient.kt similarity index 96% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/helm/HelmArtifactPushClient.kt index 2e57791dc0..f28f302675 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/helm/HelmArtifactPushClient.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.remote.type.helm +package com.tencent.bkrepo.replication.replica.repository.remote.type.helm import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.MediaTypes @@ -38,9 +38,9 @@ import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.remote.DefaultHandlerResult import com.tencent.bkrepo.replication.pojo.remote.RequestProperty -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.replica.base.handler.DefaultHandler -import com.tencent.bkrepo.replication.replica.base.impl.remote.base.PushClient +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.repository.remote.base.PushClient +import com.tencent.bkrepo.replication.util.DefaultHandler import com.tencent.bkrepo.repository.pojo.node.NodeDetail import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.MultipartBody diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmAuthorizationService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/helm/HelmAuthorizationService.kt similarity index 92% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmAuthorizationService.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/helm/HelmAuthorizationService.kt index 541a0e355b..e2895e2639 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmAuthorizationService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/helm/HelmAuthorizationService.kt @@ -25,10 +25,10 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.remote.type.helm +package com.tencent.bkrepo.replication.replica.repository.remote.type.helm import com.tencent.bkrepo.replication.pojo.remote.RequestProperty -import com.tencent.bkrepo.replication.replica.base.impl.remote.base.AuthorizationService +import com.tencent.bkrepo.replication.replica.repository.remote.base.AuthorizationService import okhttp3.OkHttpClient import org.springframework.stereotype.Component diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciArtifactPushClient.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/oci/OciArtifactPushClient.kt similarity index 95% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciArtifactPushClient.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/oci/OciArtifactPushClient.kt index 605b8811a1..c6e8f0ddda 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciArtifactPushClient.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/oci/OciArtifactPushClient.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.remote.type.oci +package com.tencent.bkrepo.replication.replica.repository.remote.type.oci import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.HttpStatus @@ -44,17 +44,17 @@ import com.tencent.bkrepo.replication.constant.OCI_MANIFEST_JSON_FULL_PATH import com.tencent.bkrepo.replication.constant.OCI_MANIFEST_URL import com.tencent.bkrepo.replication.constant.RETRY_COUNT import com.tencent.bkrepo.replication.enums.WayOfPushArtifact +import com.tencent.bkrepo.replication.exception.ArtifactPushException import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.docker.OciResponse import com.tencent.bkrepo.replication.pojo.remote.DefaultHandlerResult import com.tencent.bkrepo.replication.pojo.remote.RequestProperty -import com.tencent.bkrepo.replication.replica.base.context.FilePushContext -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.replica.base.executor.OciThreadPoolExecutor -import com.tencent.bkrepo.replication.replica.base.handler.DefaultHandler -import com.tencent.bkrepo.replication.replica.base.handler.RemoteClusterArtifactReplicationHandler -import com.tencent.bkrepo.replication.replica.base.impl.remote.base.PushClient -import com.tencent.bkrepo.replication.replica.base.impl.remote.exception.ArtifactPushException +import com.tencent.bkrepo.replication.replica.replicator.base.remote.RemoteClusterArtifactReplicationHandler +import com.tencent.bkrepo.replication.replica.repository.remote.base.PushClient +import com.tencent.bkrepo.replication.replica.context.FilePushContext +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.executor.OciThreadPoolExecutor +import com.tencent.bkrepo.replication.util.DefaultHandler import com.tencent.bkrepo.replication.util.ManifestParser import com.tencent.bkrepo.repository.pojo.node.NodeDetail import okhttp3.Headers diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/oci/OciAuthorizationService.kt similarity index 95% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/oci/OciAuthorizationService.kt index 989ac50cf1..272a22a428 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/repository/remote/type/oci/OciAuthorizationService.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.impl.remote.type.oci +package com.tencent.bkrepo.replication.replica.repository.remote.type.oci import com.tencent.bkrepo.common.api.constant.BEARER_AUTH_PREFIX import com.tencent.bkrepo.common.api.constant.HttpHeaders @@ -38,8 +38,8 @@ import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.replication.pojo.docker.OciResponse import com.tencent.bkrepo.replication.pojo.remote.BearerToken import com.tencent.bkrepo.replication.pojo.remote.RequestProperty -import com.tencent.bkrepo.replication.replica.base.impl.remote.base.AuthorizationService -import com.tencent.bkrepo.replication.replica.base.impl.remote.exception.ArtifactPushException +import com.tencent.bkrepo.replication.replica.repository.remote.base.AuthorizationService +import com.tencent.bkrepo.replication.exception.ArtifactPushException import com.tencent.bkrepo.replication.util.HttpUtils import okhttp3.OkHttpClient import okhttp3.Response diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/AbstractReplicaService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/AbstractReplicaService.kt similarity index 90% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/AbstractReplicaService.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/AbstractReplicaService.kt index 2254c2855e..cd534dfab2 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/AbstractReplicaService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/AbstractReplicaService.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base +package com.tencent.bkrepo.replication.replica.type import com.google.common.base.Throwables import com.tencent.bkrepo.common.api.pojo.ClusterNodeType @@ -40,8 +40,8 @@ import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskInfo import com.tencent.bkrepo.replication.pojo.task.objects.PackageConstraint import com.tencent.bkrepo.replication.pojo.task.objects.PathConstraint import com.tencent.bkrepo.replication.pojo.task.setting.ErrorStrategy -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.replica.base.context.ReplicaExecutionContext +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.context.ReplicaExecutionContext import com.tencent.bkrepo.replication.service.ReplicaRecordService import com.tencent.bkrepo.replication.util.ReplicationMetricsRecordUtil.convertToReplicationRecordDetailMetricsRecord import com.tencent.bkrepo.replication.util.ReplicationMetricsRecordUtil.toJson @@ -63,6 +63,53 @@ abstract class AbstractReplicaService( private val localDataManager: LocalDataManager ) : ReplicaService { + /** + * 同步 task Object + */ + protected fun replicaTaskObjects(replicaContext: ReplicaContext) { + with(replicaContext) { + // 检查版本 + replicator.checkVersion(this) + if (task.setting.automaticCreateRemoteRepo) { + // 同步项目信息 + replicator.replicaProject(replicaContext) + // 同步仓库信息 + replicator.replicaRepo(replicaContext) + } + // 按仓库同步 + if (includeAllData(this)) { + replicaByRepo(this) + return + } + replicaTaskObjectConstraints(this) + } + } + + + /** + * 判断是否包含所有仓库数据,进行仓库同步 + */ + open fun includeAllData(context: ReplicaContext): Boolean { + return false + } + + /** + * 同步task object 中的包列表或者paths + */ + open fun replicaTaskObjectConstraints(replicaContext: ReplicaContext) { + with(replicaContext) { + // 按包同步 + taskObject.packageConstraints.orEmpty().forEach { + replicaByPackageConstraint(this, it) + } + // 按路径同步 + taskObject.pathConstraints.orEmpty().forEach { + replicaByPathConstraint(this, it) + } + } + } + + /** * 同步整个仓库数据 */ diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/ReplicaService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/ReplicaService.kt similarity index 89% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/ReplicaService.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/ReplicaService.kt index 0e38fcb5fc..e71e9f392a 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/ReplicaService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/ReplicaService.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,9 +25,9 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base +package com.tencent.bkrepo.replication.replica.type -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.context.ReplicaContext /** * 同步服务接口 diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgeNodeReportJob.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeNodeReportJob.kt similarity index 96% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgeNodeReportJob.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeNodeReportJob.kt index 618315a70e..54b30780bc 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgeNodeReportJob.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeNodeReportJob.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.edge +package com.tencent.bkrepo.replication.replica.type.edge import com.tencent.bkrepo.common.service.cluster.ClusterProperties import com.tencent.bkrepo.common.service.cluster.CommitEdgeEdgeCondition diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgePullReplicaExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaExecutor.kt similarity index 98% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgePullReplicaExecutor.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaExecutor.kt index d7fdb65d56..b627606995 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgePullReplicaExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaExecutor.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.edge +package com.tencent.bkrepo.replication.replica.type.edge import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.util.toJsonString @@ -53,7 +53,7 @@ import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskInfo import com.tencent.bkrepo.replication.pojo.task.objects.ReplicaObjectInfo import com.tencent.bkrepo.replication.pojo.task.setting.ConflictStrategy import com.tencent.bkrepo.replication.pojo.task.setting.ErrorStrategy -import com.tencent.bkrepo.replication.replica.base.OkHttpClientPool +import com.tencent.bkrepo.replication.util.OkHttpClientPool import com.tencent.bkrepo.replication.replica.base.interceptor.SignInterceptor import com.tencent.bkrepo.replication.service.ReplicaRecordService import com.tencent.bkrepo.repository.api.ProjectClient diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgePullReplicaTaskEvent.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaTaskEvent.kt similarity index 96% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgePullReplicaTaskEvent.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaTaskEvent.kt index 23c002bba6..e4c3670ead 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgePullReplicaTaskEvent.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaTaskEvent.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.edge +package com.tencent.bkrepo.replication.replica.type.edge data class EdgePullReplicaTaskEvent( val taskId: String, diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgePullReplicaTaskJob.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaTaskJob.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgePullReplicaTaskJob.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaTaskJob.kt index 61262be712..3b71958920 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgePullReplicaTaskJob.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgePullReplicaTaskJob.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.edge +package com.tencent.bkrepo.replication.replica.type.edge import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID import com.tencent.bkrepo.common.service.cluster.ClusterProperties @@ -38,7 +38,7 @@ import com.tencent.bkrepo.replication.model.TReplicaTask import com.tencent.bkrepo.replication.pojo.request.ReplicaType import com.tencent.bkrepo.replication.pojo.task.ReplicaStatus import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskInfo -import com.tencent.bkrepo.replication.replica.base.executor.EdgePullThreadPoolExecutor +import com.tencent.bkrepo.replication.replica.executor.EdgePullThreadPoolExecutor import net.javacrumbs.shedlock.core.LockConfiguration import net.javacrumbs.shedlock.core.LockingTaskExecutor import org.slf4j.LoggerFactory diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgeReplicaTaskJob.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt similarity index 96% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgeReplicaTaskJob.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt index 48ca0a2f48..197e8a9d6c 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/edge/EdgeReplicaTaskJob.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.edge +package com.tencent.bkrepo.replication.replica.type.edge import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.pojo.Response @@ -42,9 +42,9 @@ import com.tencent.bkrepo.replication.api.cluster.ClusterReplicaTaskClient import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus import com.tencent.bkrepo.replication.pojo.task.EdgeReplicaTaskRecord -import com.tencent.bkrepo.replication.replica.base.OkHttpClientPool -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.replica.base.executor.ManualThreadPoolExecutor +import com.tencent.bkrepo.replication.util.OkHttpClientPool +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.executor.ManualThreadPoolExecutor import com.tencent.bkrepo.replication.replica.base.interceptor.SignInterceptor import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/ArtifactEventConsumer.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/ArtifactEventConsumer.kt similarity index 92% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/ArtifactEventConsumer.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/ArtifactEventConsumer.kt index 32bd4aa5b1..8f834e5de2 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/ArtifactEventConsumer.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/ArtifactEventConsumer.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,12 +25,12 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.event +package com.tencent.bkrepo.replication.replica.type.event import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.common.artifact.event.base.EventType import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace -import com.tencent.bkrepo.replication.replica.base.executor.EventConsumerThreadPoolExecutor +import com.tencent.bkrepo.replication.replica.executor.EventConsumerThreadPoolExecutor import com.tencent.bkrepo.replication.service.ReplicaTaskService import org.springframework.stereotype.Component diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/BootAssemblyEventListener.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/BootAssemblyEventListener.kt similarity index 95% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/BootAssemblyEventListener.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/BootAssemblyEventListener.kt index 3cfe27ff78..77184735ee 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/BootAssemblyEventListener.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/BootAssemblyEventListener.kt @@ -25,14 +25,14 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.event +package com.tencent.bkrepo.replication.replica.type.event import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.common.artifact.event.node.NodeCreatedEvent import com.tencent.bkrepo.common.artifact.event.packages.VersionCreatedEvent import com.tencent.bkrepo.common.artifact.event.packages.VersionUpdatedEvent import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace -import com.tencent.bkrepo.replication.replica.base.executor.EventConsumerThreadPoolExecutor +import com.tencent.bkrepo.replication.replica.executor.EventConsumerThreadPoolExecutor import com.tencent.bkrepo.replication.service.ReplicaTaskService import org.springframework.context.event.EventListener import org.springframework.stereotype.Component diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/EventBasedReplicaJobExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/EventBasedReplicaJobExecutor.kt similarity index 93% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/EventBasedReplicaJobExecutor.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/EventBasedReplicaJobExecutor.kt index dd26e2ef3e..582f894310 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/EventBasedReplicaJobExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/EventBasedReplicaJobExecutor.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,14 +25,14 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.event +package com.tencent.bkrepo.replication.replica.type.event import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.record.ReplicaRecordInfo import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskDetail -import com.tencent.bkrepo.replication.replica.base.executor.AbstractReplicaJobExecutor +import com.tencent.bkrepo.replication.replica.executor.AbstractReplicaJobExecutor import com.tencent.bkrepo.replication.service.ClusterNodeService import com.tencent.bkrepo.replication.service.ReplicaRecordService import org.slf4j.LoggerFactory diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/EventBasedReplicaService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/EventBasedReplicaService.kt similarity index 90% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/EventBasedReplicaService.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/EventBasedReplicaService.kt index b7759aecc4..3b1be75afe 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/EventBasedReplicaService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/EventBasedReplicaService.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.event +package com.tencent.bkrepo.replication.replica.type.event import com.tencent.bkrepo.common.api.pojo.ClusterNodeType import com.tencent.bkrepo.common.artifact.event.base.EventType @@ -35,8 +35,8 @@ import com.tencent.bkrepo.replication.constant.RETRY_COUNT import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.task.objects.PackageConstraint import com.tencent.bkrepo.replication.pojo.task.objects.PathConstraint -import com.tencent.bkrepo.replication.replica.base.AbstractReplicaService -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.type.AbstractReplicaService +import com.tencent.bkrepo.replication.replica.context.ReplicaContext import com.tencent.bkrepo.replication.service.ReplicaRecordService import org.springframework.stereotype.Component @@ -53,7 +53,9 @@ class EventBasedReplicaService( with(context) { // 同步仓库 retry(times = RETRY_COUNT, delayInSeconds = DELAY_IN_SECONDS) { - replicator.replicaRepo(this) + if (task.setting.automaticCreateRemoteRepo) { + replicator.replicaRepo(this) + } } when (event.type) { EventType.NODE_CREATED -> { diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/EventConsumer.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/EventConsumer.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/EventConsumer.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/EventConsumer.kt index 1c97bc725f..f5c63981b6 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/event/EventConsumer.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/event/EventConsumer.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.event +package com.tencent.bkrepo.replication.replica.type.event import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.common.artifact.event.base.EventType diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/manual/ManualBasedReplicaService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/manual/ManualBasedReplicaService.kt similarity index 51% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/manual/ManualBasedReplicaService.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/manual/ManualBasedReplicaService.kt index a31f732ec0..c7cb865ece 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/manual/ManualBasedReplicaService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/manual/ManualBasedReplicaService.kt @@ -25,14 +25,17 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.manual +package com.tencent.bkrepo.replication.replica.type.manual import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager -import com.tencent.bkrepo.replication.replica.base.AbstractReplicaService -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.replica.base.executor.ManualThreadPoolExecutor +import com.tencent.bkrepo.replication.pojo.request.ReplicaObjectType +import com.tencent.bkrepo.replication.pojo.task.objects.PackageConstraint +import com.tencent.bkrepo.replication.pojo.task.objects.PathConstraint +import com.tencent.bkrepo.replication.replica.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.executor.ManualThreadPoolExecutor +import com.tencent.bkrepo.replication.replica.type.AbstractReplicaService import com.tencent.bkrepo.replication.service.ReplicaRecordService import org.springframework.stereotype.Component import java.util.concurrent.Callable @@ -50,40 +53,70 @@ class ManualBasedReplicaService( ) : AbstractReplicaService(replicaRecordService, localDataManager) { private val executor = ManualThreadPoolExecutor.instance override fun replica(context: ReplicaContext) { - val semaphore = Semaphore(replicationProperties.manualConcurrencyNum) - with(context) { + replicaTaskObjects(context) + } + + /** + * 是否包含所有仓库数据 + */ + override fun includeAllData(context: ReplicaContext): Boolean { + return context.taskObject.packageConstraints.isNullOrEmpty() && + context.taskObject.pathConstraints.isNullOrEmpty() && + context.task.replicaObjectType == ReplicaObjectType.REPOSITORY + } + + /** + * 同步task object 中的包列表或者paths + */ + override fun replicaTaskObjectConstraints(replicaContext: ReplicaContext) { + with(replicaContext) { + val semaphore = Semaphore(replicationProperties.manualConcurrencyNum) + // 按包同步 val futureList = mutableListOf>() - taskObject.packageConstraints.orEmpty().forEach { - semaphore.acquire() - futureList.add( - executor.submit( - Callable{ - try { - replicaByPackageConstraint(this, it) - } finally { - semaphore.release() - } - }.trace() - ) - ) - } + // 按包同步 + mapEachTaskObject(semaphore, futureList, taskObject.packageConstraints.orEmpty(), replicaContext) // 按路径同步 - taskObject.pathConstraints.orEmpty().forEach { - semaphore.acquire() - futureList.add( - executor.submit( - Callable { - try { - replicaByPathConstraint(this, it) - } finally { - semaphore.release() - } - }.trace() - ) - ) - } + mapEachTaskObject(semaphore, futureList, taskObject.pathConstraints.orEmpty(), replicaContext) + futureList.forEach { it.get() } } } + + + /** + * 遍历执行所有分发任务 + */ + private fun mapEachTaskObject( + semaphore: Semaphore, futureList: MutableList>, + taskObjects: List, context: ReplicaContext + ) { + for (taskObject in taskObjects) { + semaphore.acquire() + futureList.add( + executor.submit( + Callable{ + try { + replicaTaskObject(context, taskObject) + } finally { + semaphore.release() + } + }.trace() + ) + ) + } + } + + /** + * 分发具体内容 + */ + private fun replicaTaskObject(replicaContext: ReplicaContext, constraint: Any) { + if (constraint is PathConstraint) { + replicaByPathConstraint(replicaContext, constraint) + } + if (constraint is PackageConstraint) { + replicaByPackageConstraint(replicaContext, constraint) + } + } + } diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/manual/ManualReplicaJobExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/manual/ManualReplicaJobExecutor.kt similarity index 97% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/manual/ManualReplicaJobExecutor.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/manual/ManualReplicaJobExecutor.kt index a997ebf55c..1b099b6057 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/manual/ManualReplicaJobExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/manual/ManualReplicaJobExecutor.kt @@ -25,14 +25,14 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.manual +package com.tencent.bkrepo.replication.replica.type.manual import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus import com.tencent.bkrepo.replication.pojo.task.ReplicaStatus import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskDetail -import com.tencent.bkrepo.replication.replica.base.executor.AbstractReplicaJobExecutor +import com.tencent.bkrepo.replication.replica.executor.AbstractReplicaJobExecutor import com.tencent.bkrepo.replication.service.ClusterNodeService import com.tencent.bkrepo.replication.service.ReplicaRecordService import com.tencent.bkrepo.replication.service.impl.ReplicaRecordServiceImpl.Companion.isCronJob diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ReplicaTaskScheduler.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ReplicaTaskScheduler.kt similarity index 96% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ReplicaTaskScheduler.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ReplicaTaskScheduler.kt index 78838c1db9..5d3a12091c 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ReplicaTaskScheduler.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ReplicaTaskScheduler.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.schedule +package com.tencent.bkrepo.replication.replica.type.schedule import org.quartz.JobDetail import org.quartz.JobKey diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ScheduledReplicaJob.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ScheduledReplicaJob.kt similarity index 95% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ScheduledReplicaJob.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ScheduledReplicaJob.kt index 26eb88654d..e468d928bf 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ScheduledReplicaJob.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ScheduledReplicaJob.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.schedule +package com.tencent.bkrepo.replication.replica.type.schedule import com.tencent.bkrepo.common.service.util.SpringContextUtils import net.javacrumbs.shedlock.core.LockConfiguration diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ScheduledReplicaJobExecutor.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ScheduledReplicaJobExecutor.kt similarity index 96% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ScheduledReplicaJobExecutor.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ScheduledReplicaJobExecutor.kt index 3f16b7447b..c76a213e5b 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ScheduledReplicaJobExecutor.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ScheduledReplicaJobExecutor.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,13 +25,13 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.schedule +package com.tencent.bkrepo.replication.replica.type.schedule import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskInfo -import com.tencent.bkrepo.replication.replica.base.executor.AbstractReplicaJobExecutor +import com.tencent.bkrepo.replication.replica.executor.AbstractReplicaJobExecutor import com.tencent.bkrepo.replication.service.ClusterNodeService import com.tencent.bkrepo.replication.service.ReplicaRecordService import com.tencent.bkrepo.replication.service.ReplicaTaskService diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ScheduledReplicaService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ScheduledReplicaService.kt similarity index 70% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ScheduledReplicaService.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ScheduledReplicaService.kt index 79ff8f1f87..6b83a8dacf 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/ScheduledReplicaService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/ScheduledReplicaService.kt @@ -25,11 +25,11 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.schedule +package com.tencent.bkrepo.replication.replica.type.schedule import com.tencent.bkrepo.replication.manager.LocalDataManager -import com.tencent.bkrepo.replication.replica.base.AbstractReplicaService -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.type.AbstractReplicaService +import com.tencent.bkrepo.replication.replica.context.ReplicaContext import com.tencent.bkrepo.replication.service.ReplicaRecordService import org.springframework.stereotype.Component @@ -43,33 +43,13 @@ class ScheduledReplicaService( ) : AbstractReplicaService(replicaRecordService, localDataManager) { override fun replica(context: ReplicaContext) { - with(context) { - // 检查版本 - replicator.checkVersion(this) - // 同步项目 - replicator.replicaProject(this) - // 同步仓库 - replicator.replicaRepo(this) - // 按仓库同步 - if (includeAllData(this)) { - replicaByRepo(this) - return - } - // 按包同步 - taskObject.packageConstraints.orEmpty().forEach { - replicaByPackageConstraint(this, it) - } - // 按路径同步 - taskObject.pathConstraints.orEmpty().forEach { - replicaByPathConstraint(this, it) - } - } + replicaTaskObjects(context) } /** * 是否包含所有仓库数据 */ - private fun includeAllData(context: ReplicaContext): Boolean { + override fun includeAllData(context: ReplicaContext): Boolean { return context.taskObject.packageConstraints.isNullOrEmpty() && context.taskObject.pathConstraints.isNullOrEmpty() } diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/TaskReloadManager.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/TaskReloadManager.kt similarity index 94% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/TaskReloadManager.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/TaskReloadManager.kt index c056572277..19314ef843 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/schedule/TaskReloadManager.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/schedule/TaskReloadManager.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -25,13 +25,13 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.schedule +package com.tencent.bkrepo.replication.replica.type.schedule import com.tencent.bkrepo.common.service.cluster.StandaloneJob import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskInfo import com.tencent.bkrepo.replication.pojo.task.setting.ExecutionStrategy -import com.tencent.bkrepo.replication.replica.schedule.ReplicaTaskScheduler.Companion.JOB_DATA_TASK_KEY -import com.tencent.bkrepo.replication.replica.schedule.ReplicaTaskScheduler.Companion.REPLICA_JOB_GROUP +import com.tencent.bkrepo.replication.replica.type.schedule.ReplicaTaskScheduler.Companion.JOB_DATA_TASK_KEY +import com.tencent.bkrepo.replication.replica.type.schedule.ReplicaTaskScheduler.Companion.REPLICA_JOB_GROUP import com.tencent.bkrepo.replication.service.ReplicaTaskService import org.quartz.CronScheduleBuilder import org.quartz.JobBuilder diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/EdgeReplicaTaskRecordService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/EdgeReplicaTaskRecordService.kt index a317f5b362..aa6705b8c6 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/EdgeReplicaTaskRecordService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/EdgeReplicaTaskRecordService.kt @@ -29,7 +29,7 @@ package com.tencent.bkrepo.replication.service import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus import com.tencent.bkrepo.replication.pojo.task.EdgeReplicaTaskRecord -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.context.ReplicaContext import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/EdgeReplicaTaskRecordServiceImpl.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/EdgeReplicaTaskRecordServiceImpl.kt index 9c61728148..d0c3c649b8 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/EdgeReplicaTaskRecordServiceImpl.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/EdgeReplicaTaskRecordServiceImpl.kt @@ -35,7 +35,7 @@ import com.tencent.bkrepo.replication.exception.ReplicationMessageCode import com.tencent.bkrepo.replication.model.TEdgeReplicaTaskRecord import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus import com.tencent.bkrepo.replication.pojo.task.EdgeReplicaTaskRecord -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext +import com.tencent.bkrepo.replication.replica.context.ReplicaContext import com.tencent.bkrepo.replication.service.EdgeReplicaTaskRecordService import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.packages.PackageSummary diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt index 4cb6ffba58..0b08923d27 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt @@ -61,9 +61,9 @@ import com.tencent.bkrepo.replication.pojo.task.objects.ReplicaObjectInfo import com.tencent.bkrepo.replication.pojo.task.request.ReplicaTaskCreateRequest import com.tencent.bkrepo.replication.pojo.task.request.ReplicaTaskUpdateRequest import com.tencent.bkrepo.replication.pojo.task.setting.ConflictStrategy -import com.tencent.bkrepo.replication.replica.base.executor.RunOnceThreadPoolExecutor -import com.tencent.bkrepo.replication.replica.event.EventBasedReplicaJobExecutor -import com.tencent.bkrepo.replication.replica.manual.ManualReplicaJobExecutor +import com.tencent.bkrepo.replication.replica.executor.RunOnceThreadPoolExecutor +import com.tencent.bkrepo.replication.replica.type.event.EventBasedReplicaJobExecutor +import com.tencent.bkrepo.replication.replica.type.manual.ManualReplicaJobExecutor import com.tencent.bkrepo.replication.service.ClusterNodePermissionService import com.tencent.bkrepo.replication.service.ClusterNodeService import com.tencent.bkrepo.replication.service.RemoteNodeService @@ -99,24 +99,7 @@ class RemoteNodeServiceImpl( ): List { return requests.configs.map { validateName(it.name) - val realName = NAME.format(projectId, repoName, it.name) - val tClusterNode = if (it.clusterId.isNullOrEmpty()) { - val oldCluster = clusterNodeService.getByClusterName(realName) - if (oldCluster == null) { - clusterNodeService.create( - SecurityUtils.getUserId(), buildClusterNodeCreateRequest(projectId, repoName, it) - ) - } else { - val updateRequest = convertCreateToUpdate(it) - clusterNodeService.update( - buildClusterNodeUpdateRequest(projectId, repoName, it.name, updateRequest) - ) - } - } else { - clusterNodeService.getByClusterId(it.clusterId!!) - ?: throw ErrorCodeException(ReplicationMessageCode.CLUSTER_NODE_NOT_FOUND, it.clusterId!!) - } - + val tClusterNode = createClusterInfo(projectId, repoName, it) createOrUpdateTask( projectId = projectId, repoName = repoName, @@ -231,34 +214,6 @@ class RemoteNodeServiceImpl( } - private fun createWithRemoteClient( - projectId: String, repoName: String, - request: RemoteRunOnceTaskCreateRequest - ): Boolean { - val host = if (!request.clusterId.isNullOrEmpty()) { - val clusterInfo = clusterNodeService.getByClusterId(request.clusterId!!) - ?: throw ErrorCodeException(ReplicationMessageCode.CLUSTER_NODE_NOT_FOUND, request.clusterId!!) - clusterInfo.url - } else { - addProtocol(request.registry!!).toString() - } - buildExecuteClientWithHost(host)?.let { - try{ - it.createRunOnceTask(projectId, repoName, request) - return true - } catch (e: Exception) { - // fegin连不上时需要降级为本地执行 - logger.warn("Cloud not run task on remote node, will run with current node") - } - } - return false - } - - private fun buildExecuteClientWithHost(host: String) : ReplicaTaskOperationClient? { - return replicaNodeDispatchService.findReplicaClientByHost( - host, ReplicaTaskOperationClient::class.java - ) - } // dispatch 默认为TRUE, fegin调用时为FALSE,避免feign调用时再次进行配置判断 override fun executeRunOnceTask(projectId: String, repoName: String, name: String, dispatch: Boolean) { @@ -273,34 +228,6 @@ class RemoteNodeServiceImpl( } - /** - * 执行成功返回true,fegin连不上时需要降级为本地执行,返回false - */ - private fun executeWithRemoteClient( - projectId: String, repoName: String, name: String, - taskDetail: ReplicaTaskDetail - ): Boolean { - buildExecuteClient(taskDetail)?.let { - try{ - it.executeRunOnceTask(projectId, repoName, name) - return true - } catch (e: Exception) { - // fegin连不上时需要降级为本地执行 - logger.warn("Cloud not run task on remote node, will run with current node") - } - } - return false - } - - - private fun buildExecuteClient( - taskDetail: ReplicaTaskDetail - ) : ReplicaTaskOperationClient? { - return replicaNodeDispatchService.findReplicaClient( - taskDetail, ReplicaTaskOperationClient::class.java - ) - } - override fun getRunOnceTaskResult(projectId: String, repoName: String, name: String): ReplicaRecordInfo? { val taskInfo = getReplicaTaskInfo(projectId, repoName, name) val record = replicaRecordService.findLatestRecord(taskInfo.key) @@ -323,38 +250,35 @@ class RemoteNodeServiceImpl( deleteByTaskName(taskName) } - private fun deleteByTaskName(taskName: String) { - logger.info("Task $taskName will be deleted!") - replicaTaskService.getByTaskName(taskName)?.let { - if (it.status!! != ReplicaStatus.COMPLETED && it.replicaType != ReplicaType.RUN_ONCE) { - logger.warn("The name $taskName of runonce task is still running") - throw ErrorCodeException(CommonMessageCode.REQUEST_DENIED, taskName) - } - clusterNodeService.getByClusterName(taskName)?.let { node -> - clusterNodeService.deleteById(node.id!!) - } - replicaTaskService.deleteByTaskKey(it.key) - } - } - - private fun getTaskDetail( + /** + * 根据分发任务内容创建对应的集群信息 + */ + private fun createClusterInfo( projectId: String, repoName: String, - name: String - ): ReplicaTaskDetail { - val taskInfo = getReplicaTaskInfo(projectId, repoName, name) - return replicaTaskService.getDetailByTaskKey(taskInfo.key) + request: RemoteConfigCreateRequest + ): ClusterNodeInfo { + val realName = NAME.format(projectId, repoName, request.name) + // clusterId为空的情况默认为remote集群 + return if (request.clusterId.isNullOrEmpty()) { + val oldCluster = clusterNodeService.getByClusterName(realName) + if (oldCluster == null) { + clusterNodeService.create( + SecurityUtils.getUserId(), buildClusterNodeCreateRequest(projectId, repoName, request) + ) + } else { + val updateRequest = convertCreateToUpdate(request) + clusterNodeService.update( + buildClusterNodeUpdateRequest(projectId, repoName, request.name, updateRequest) + ) + } + } else { + // clusterId不为空则任务是通过集群,同构集群都是在任务创建前已经创建 + clusterNodeService.getByClusterId(request.clusterId!!) + ?: throw ErrorCodeException(ReplicationMessageCode.CLUSTER_NODE_NOT_FOUND, request.clusterId!!) + } } - private fun getReplicaTaskInfo( - projectId: String, - repoName: String, - name: String - ): ReplicaTaskInfo { - val realName = NAME.format(projectId, repoName, name) - return replicaTaskService.getByTaskName(realName) - ?: throw ErrorCodeException(CommonMessageCode.RESOURCE_NOT_FOUND, name) - } /** * 当远端集群创建后,创建/更新对应的任务 @@ -372,29 +296,12 @@ class RemoteNodeServiceImpl( ) val repositoryDetail = localDataManager.findRepoByName(projectId, repoName) - if (pathConstraints.isNullOrEmpty() && packageConstraints.isNullOrEmpty() - && replicaType == ReplicaType.RUN_ONCE) { - throw ErrorCodeException(CommonMessageCode.PARAMETER_INVALID, "Package or path") - } - val replicaTaskObjects = listOf( - ReplicaObjectInfo( - localRepoName = repoName, - repoType = repositoryDetail.type, - remoteProjectId = remoteProjectId, - remoteRepoName = remoteRepoName, - packageConstraints = packageConstraints, - pathConstraints = pathConstraints - ) + // 只有在同构集群内部才可以通过一次性分发任务进行仓库同步 + validateReplicaObject(request, clusterInfo) + val replicaObjectType = getReplicaObjectType(request) + val replicaTaskObjects = buildReplicaTaskObjects( + repoName, repositoryDetail.type, replicaObjectType, request ) - val replicaObjectType = if (replicaType == ReplicaType.RUN_ONCE) { - if (!pathConstraints.isNullOrEmpty()) { - ReplicaObjectType.PATH - } else { - ReplicaObjectType.PACKAGE - } - } else { - ReplicaObjectType.REPOSITORY - } var task = replicaTaskService.getByTaskName(NAME.format(projectId, repoName, name)) if (task == null) { val taskCreateRequest = ReplicaTaskCreateRequest( @@ -439,6 +346,72 @@ class RemoteNodeServiceImpl( } } + /** + * 校验分发内容 + */ + private fun validateReplicaObject( + request: RemoteConfigCreateRequest, + clusterInfo: ClusterNodeInfo + ) { + with(request) { + // 兼容历史数据,replicaObjectType为空的情况 + if (pathConstraints.isNullOrEmpty() && packageConstraints.isNullOrEmpty() && + clusterInfo.type == ClusterNodeType.REMOTE) { + throw ErrorCodeException(CommonMessageCode.PARAMETER_INVALID, "Package or path") + } + + if (pathConstraints.isNullOrEmpty() && packageConstraints.isNullOrEmpty() && + replicaObjectType != ReplicaObjectType.REPOSITORY) { + throw ErrorCodeException(CommonMessageCode.PARAMETER_INVALID, "Package or path") + } + } + } + + + /** + * 获取对应分发任务的内容类型 + */ + private fun getReplicaObjectType(request: RemoteConfigCreateRequest): ReplicaObjectType { + with(request) { + if (replicaObjectType != null) return replicaObjectType!! + return if (replicaType == ReplicaType.RUN_ONCE) { + if (!pathConstraints.isNullOrEmpty()) { + ReplicaObjectType.PATH + } else { + ReplicaObjectType.PACKAGE + } + } else { + ReplicaObjectType.REPOSITORY + } + } + } + + private fun buildReplicaTaskObjects( + repoName: String, + repoType: RepositoryType, + replicaObjectType: ReplicaObjectType, + request: RemoteConfigCreateRequest + ): List { + with(request) { + val taskObjects = mutableListOf() + val (packageConstraints, pathConstraints) = if (replicaObjectType == ReplicaObjectType.REPOSITORY) { + Pair(null, null) + } else { + Pair(packageConstraints, pathConstraints) + } + taskObjects.add(ReplicaObjectInfo( + localRepoName = repoName, + repoType = repoType, + remoteProjectId = remoteProjectId, + remoteRepoName = remoteRepoName, + packageConstraints = packageConstraints, + pathConstraints = pathConstraints + )) + return taskObjects + } + } + + private fun getRemoteInfoByName( projectId: String, repoName: String, @@ -456,6 +429,103 @@ class RemoteNodeServiceImpl( } } + private fun getTaskDetail( + projectId: String, + repoName: String, + name: String + ): ReplicaTaskDetail { + val taskInfo = getReplicaTaskInfo(projectId, repoName, name) + return replicaTaskService.getDetailByTaskKey(taskInfo.key) + } + + + + private fun createWithRemoteClient( + projectId: String, repoName: String, + request: RemoteRunOnceTaskCreateRequest + ): Boolean { + val host = if (!request.clusterId.isNullOrEmpty()) { + val clusterInfo = clusterNodeService.getByClusterId(request.clusterId!!) + ?: throw ErrorCodeException(ReplicationMessageCode.CLUSTER_NODE_NOT_FOUND, request.clusterId!!) + clusterInfo.url + } else { + addProtocol(request.registry!!).toString() + } + buildExecuteClientWithHost(host)?.let { + try{ + it.createRunOnceTask(projectId, repoName, request) + return true + } catch (e: Exception) { + // fegin连不上时需要降级为本地执行 + logger.warn("Cloud not run task on remote node, will run with current node") + } + } + return false + } + + private fun buildExecuteClientWithHost(host: String) : ReplicaTaskOperationClient? { + return replicaNodeDispatchService.findReplicaClientByHost( + host, ReplicaTaskOperationClient::class.java + ) + } + + + /** + * 执行成功返回true,fegin连不上时需要降级为本地执行,返回false + */ + private fun executeWithRemoteClient( + projectId: String, repoName: String, name: String, + taskDetail: ReplicaTaskDetail + ): Boolean { + buildExecuteClient(taskDetail)?.let { + try{ + it.executeRunOnceTask(projectId, repoName, name) + return true + } catch (e: Exception) { + // fegin连不上时需要降级为本地执行 + logger.warn("Cloud not run task on remote node, will run with current node") + } + } + return false + } + + + private fun buildExecuteClient( + taskDetail: ReplicaTaskDetail + ) : ReplicaTaskOperationClient? { + return replicaNodeDispatchService.findReplicaClient( + taskDetail, ReplicaTaskOperationClient::class.java + ) + } + + + + private fun getReplicaTaskInfo( + projectId: String, + repoName: String, + name: String + ): ReplicaTaskInfo { + val realName = NAME.format(projectId, repoName, name) + return replicaTaskService.getByTaskName(realName) + ?: throw ErrorCodeException(CommonMessageCode.RESOURCE_NOT_FOUND, name) + } + + + private fun deleteByTaskName(taskName: String) { + logger.info("Task $taskName will be deleted!") + replicaTaskService.getByTaskName(taskName)?.let { + if (it.status!! != ReplicaStatus.COMPLETED && it.replicaType != ReplicaType.RUN_ONCE) { + logger.warn("The name $taskName of runonce task is still running") + throw ErrorCodeException(CommonMessageCode.REQUEST_DENIED, taskName) + } + clusterNodeService.getByClusterName(taskName)?.let { node -> + clusterNodeService.deleteById(node.id!!) + } + replicaTaskService.deleteByTaskKey(it.key) + } + } + + private fun buildClusterNodeCreateRequest( projectId: String, repoName: String, @@ -530,6 +600,7 @@ class RemoteNodeServiceImpl( remoteUserPassword = remoteUserPassword, remoteProjectId = remoteProjectId, remoteRepoName = remoteRepoName, + replicaObjectType = replicaObjectType, packageConstraints = packageConstraints, pathConstraints = pathConstraints, replicaType = replicaType, @@ -551,6 +622,7 @@ class RemoteNodeServiceImpl( remoteUserUsername = remoteUserUsername, remoteUserPassword = remoteUserPassword, packageConstraints = packageConstraints, + replicaObjectType = replicaObjectType, pathConstraints = pathConstraints, replicaType = replicaType, setting = setting, @@ -577,6 +649,7 @@ class RemoteNodeServiceImpl( remoteUserPassword = remoteUserPassword, remoteProjectId = remoteProjectId, remoteRepoName = remoteRepoName, + replicaObjectType = replicaObjectType, packageConstraints = packageConstraints, pathConstraints = pathConstraints, replicaType = replicaType, diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaExtServiceImpl.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaExtServiceImpl.kt index 0b62dfb9b3..2e9430f19e 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaExtServiceImpl.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaExtServiceImpl.kt @@ -56,9 +56,9 @@ import com.tencent.bkrepo.replication.pojo.task.objects.PackageConstraint import com.tencent.bkrepo.replication.pojo.task.objects.PathConstraint import com.tencent.bkrepo.replication.pojo.task.setting.ConflictStrategy import com.tencent.bkrepo.replication.pojo.task.setting.ReplicaSetting -import com.tencent.bkrepo.replication.replica.base.OkHttpClientPool -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext.Companion.READ_TIMEOUT -import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext.Companion.WRITE_TIMEOUT +import com.tencent.bkrepo.replication.util.OkHttpClientPool +import com.tencent.bkrepo.replication.replica.context.ReplicaContext.Companion.READ_TIMEOUT +import com.tencent.bkrepo.replication.replica.context.ReplicaContext.Companion.WRITE_TIMEOUT import com.tencent.bkrepo.replication.service.RemoteNodeService import com.tencent.bkrepo.replication.service.ReplicaExtService import com.tencent.bkrepo.repository.api.NodeClient diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaTaskServiceImpl.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaTaskServiceImpl.kt index d8332ac286..01f651d857 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaTaskServiceImpl.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/ReplicaTaskServiceImpl.kt @@ -56,11 +56,11 @@ import com.tencent.bkrepo.replication.pojo.task.request.ReplicaTaskCreateRequest import com.tencent.bkrepo.replication.pojo.task.request.ReplicaTaskUpdateRequest import com.tencent.bkrepo.replication.pojo.task.request.TaskPageParam import com.tencent.bkrepo.replication.pojo.task.setting.ExecutionStrategy -import com.tencent.bkrepo.replication.replica.edge.EdgePullReplicaExecutor -import com.tencent.bkrepo.replication.replica.schedule.ReplicaTaskScheduler -import com.tencent.bkrepo.replication.replica.schedule.ReplicaTaskScheduler.Companion.JOB_DATA_TASK_KEY -import com.tencent.bkrepo.replication.replica.schedule.ReplicaTaskScheduler.Companion.REPLICA_JOB_GROUP -import com.tencent.bkrepo.replication.replica.schedule.ScheduledReplicaJob +import com.tencent.bkrepo.replication.replica.type.edge.EdgePullReplicaExecutor +import com.tencent.bkrepo.replication.replica.type.schedule.ReplicaTaskScheduler +import com.tencent.bkrepo.replication.replica.type.schedule.ReplicaTaskScheduler.Companion.JOB_DATA_TASK_KEY +import com.tencent.bkrepo.replication.replica.type.schedule.ReplicaTaskScheduler.Companion.REPLICA_JOB_GROUP +import com.tencent.bkrepo.replication.replica.type.schedule.ScheduledReplicaJob import com.tencent.bkrepo.replication.service.ClusterNodeService import com.tencent.bkrepo.replication.service.ReplicaRecordService import com.tencent.bkrepo.replication.service.ReplicaTaskService diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/DefaultHandler.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/DefaultHandler.kt similarity index 95% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/DefaultHandler.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/DefaultHandler.kt index 7b301a3ceb..f62dc089cb 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/DefaultHandler.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/DefaultHandler.kt @@ -25,15 +25,14 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.replica.base.handler +package com.tencent.bkrepo.replication.util import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.replication.exception.ArtifactPushException import com.tencent.bkrepo.replication.pojo.remote.DefaultHandlerResult import com.tencent.bkrepo.replication.pojo.remote.RequestProperty -import com.tencent.bkrepo.replication.replica.base.impl.remote.exception.ArtifactPushException -import com.tencent.bkrepo.replication.util.HttpUtils import okhttp3.OkHttpClient import okhttp3.Response diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/OkHttpClientPool.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/OkHttpClientPool.kt similarity index 50% rename from src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/OkHttpClientPool.kt rename to src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/OkHttpClientPool.kt index e6bb5caa62..d639aa9b1a 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/OkHttpClientPool.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/OkHttpClientPool.kt @@ -1,4 +1,31 @@ -package com.tencent.bkrepo.replication.replica.base +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.replication.util import com.tencent.bkrepo.common.service.cluster.ClusterInfo import com.tencent.bkrepo.common.service.util.okhttp.HttpClientBuilderFactory diff --git a/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/QuartzJobReloadTest.kt b/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/QuartzJobReloadTest.kt index 43278ca818..0799a4d68c 100644 --- a/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/QuartzJobReloadTest.kt +++ b/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/QuartzJobReloadTest.kt @@ -27,8 +27,8 @@ package com.tencent.bkrepo.replication -import com.tencent.bkrepo.replication.replica.schedule.ReplicaTaskScheduler -import com.tencent.bkrepo.replication.replica.schedule.ReplicaTaskScheduler.Companion.REPLICA_JOB_GROUP +import com.tencent.bkrepo.replication.replica.type.schedule.ReplicaTaskScheduler +import com.tencent.bkrepo.replication.replica.type.schedule.ReplicaTaskScheduler.Companion.REPLICA_JOB_GROUP import org.junit.jupiter.api.Test import org.quartz.InterruptableJob import org.quartz.JobBuilder diff --git a/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/service/TaskReloadManagerTest.kt b/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/service/TaskReloadManagerTest.kt index d340d30271..daa60fed05 100644 --- a/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/service/TaskReloadManagerTest.kt +++ b/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/service/TaskReloadManagerTest.kt @@ -42,9 +42,9 @@ import com.tencent.bkrepo.replication.pojo.task.objects.ReplicaObjectInfo import com.tencent.bkrepo.replication.pojo.task.request.ReplicaTaskCreateRequest import com.tencent.bkrepo.replication.pojo.task.setting.ExecutionPlan import com.tencent.bkrepo.replication.pojo.task.setting.ReplicaSetting -import com.tencent.bkrepo.replication.replica.schedule.ReplicaTaskScheduler -import com.tencent.bkrepo.replication.replica.schedule.ScheduledReplicaJobExecutor -import com.tencent.bkrepo.replication.replica.schedule.TaskReloadManager +import com.tencent.bkrepo.replication.replica.type.schedule.ReplicaTaskScheduler +import com.tencent.bkrepo.replication.replica.type.schedule.ScheduledReplicaJobExecutor +import com.tencent.bkrepo.replication.replica.type.schedule.TaskReloadManager import com.tencent.bkrepo.replication.service.impl.ReplicaTaskServiceImpl import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeAll